{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Toy Dataset: Sorting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "N = 1000\n",
    "source_data = []\n",
    "target_data = []\n",
    "num_data = 20000\n",
    "\n",
    "for i in range(num_data):\n",
    "    length = 15 + np.random.choice(11)\n",
    "    s = np.random.choice(N, length)\n",
    "    t = np.sort(s)\n",
    "\n",
    "    source_data.append(\" \".join(s.astype(str).tolist()))\n",
    "    target_data.append(\" \".join(t.astype(str).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data={\"0\": source_data})\n",
    "tdf = pd.DataFrame(data={\"0\": target_data})\n",
    "\n",
    "if not os.path.exists(\"./example/\"):\n",
    "    os.makedirs(\"./example/\")\n",
    "\n",
    "df.to_csv(\"./example/train_source.txt\", header=None, index=None)\n",
    "tdf.to_csv(\"./example/train_target.txt\", header=None, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev\n",
    "dev_source_data = []\n",
    "dev_target_data = []\n",
    "dev_num_data = 1000\n",
    "\n",
    "for i in range(dev_num_data):\n",
    "    length = 15 + np.random.choice(11)\n",
    "    s = np.random.choice(N, length)\n",
    "    t = np.sort(s)\n",
    "\n",
    "    dev_source_data.append(\" \".join(s.astype(str).tolist()))\n",
    "    dev_target_data.append(\" \".join(t.astype(str).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddf = pd.DataFrame(data={\"0\": dev_source_data})\n",
    "tddf = pd.DataFrame(data={\"0\": dev_target_data})\n",
    "\n",
    "ddf.to_csv(\"./example/dev_source.txt\", header=None, index=None)\n",
    "tddf.to_csv(\"./example/dev_target.txt\", header=None, index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoder and Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_embeddings(vocab_size, embed_size, dtype=tf.float32,\n",
    "                    initializer=None, initial_values=None,\n",
    "                    name='embeddings'):\n",
    "    \"\"\"\n",
    "    embeddings:\n",
    "        initialize trainable embeddings or load pretrained from files\n",
    "    \"\"\"\n",
    "    with tf.variable_scope(name):\n",
    "        if initial_values:\n",
    "            embeddings = tf.Variable(initial_value=initial_values,\n",
    "                                     name=\"embeddings\", dtype=dtype)\n",
    "        else:\n",
    "            if initializer is None:\n",
    "                initializer = tf.contrib.layers.xavier_initializer()\n",
    "\n",
    "            embeddings = tf.Variable(\n",
    "                initializer(shape=(vocab_size, embed_size)),\n",
    "                name=\"embeddings\", dtype=dtype)\n",
    "\n",
    "        # id_0 represents SOS token, id_1 represents EOS token\n",
    "        se_embed = tf.get_variable(\"SOS/EOS\", [2, embed_size], dtype)\n",
    "        # id_2 represents constant all zeros\n",
    "        zero_embed = tf.zeros(shape=[1, embed_size])\n",
    "        embeddings = tf.concat([se_embed, zero_embed, embeddings], axis=0)\n",
    "\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoder\n",
    "from tensorflow.contrib.rnn import LSTMStateTuple\n",
    "\n",
    "def create_cell(num_units, cell_type, forget_bias=1.0):\n",
    "    \"\"\"\n",
    "    Cell: build a recurrent cell\n",
    "        num_units: number of hidden cell units\n",
    "        cell_type: LSTM, GRU, LN_LSTM (layer_normalize)\n",
    "    \"\"\"\n",
    "    if cell_type == \"LSTM\":\n",
    "        cell = tf.nn.rnn_cell.BasicLSTMCell(num_units, forget_bias=forget_bias)\n",
    "\n",
    "    elif cell_type == \"GRU\":\n",
    "        cell = tf.nn.rnn_cell.GRUCell(num_units)\n",
    "\n",
    "    elif cell_type == \"LN_LSTM\":\n",
    "        cell = tf.contrib.rnn.LayerNormBasicLSTMCell(\n",
    "            num_units,\n",
    "            forget_bias=forget_bias,\n",
    "            layer_norm=True)\n",
    "\n",
    "    else:\n",
    "        raise ValueError(\"Unknown cell type %s\" % cell_type)\n",
    "\n",
    "    return cell\n",
    "\n",
    "\n",
    "def build_rnn_cell(num_layers, num_units, cell_type, forget_bias=1.0):\n",
    "    \"\"\"\n",
    "    RNN_cell: build a multi-layer rnn cell\n",
    "        num_layers: number of hidden layers\n",
    "    \"\"\"\n",
    "    cell_seq = []\n",
    "    for i in range(num_layers):\n",
    "        cell = create_cell(num_units, cell_type, forget_bias)\n",
    "        cell_seq.append(cell)\n",
    "\n",
    "    if num_layers > 1:\n",
    "        rnn_cell = tf.nn.rnn_cell.MultiRNNCell(cell_seq)\n",
    "    else:\n",
    "        rnn_cell = cell_seq[0]\n",
    "\n",
    "    return rnn_cell\n",
    "\n",
    "\n",
    "def build_encoder(embeddings, source_ids, num_layers, num_units, cell_type,\n",
    "                  forget_bias=1.0, bidir=False, time_major=False,\n",
    "                  dtype=tf.float32, name=\"encoder\"):\n",
    "    \"\"\"\n",
    "    encoder: build rnn encoder for Seq2seq\n",
    "        source_ids: [batch_size, max_time]\n",
    "        bidir: bidirectional or unidirectional\n",
    "\n",
    "    Returns:\n",
    "        encoder_outputs: [batch_size, max_time, num_units]\n",
    "        encoder_states: (StateTuple(shape=(batch_size, num_units)), ...)\n",
    "    \"\"\"\n",
    "    with tf.variable_scope(name):\n",
    "        if time_major:\n",
    "            source_ids = tf.transpose(source_ids)\n",
    "\n",
    "        # embedding lookup, embed_inputs: [max_time, batch_size, num_units]\n",
    "        embed_inputs = tf.nn.embedding_lookup(embeddings, source_ids)\n",
    "\n",
    "        # bidirectional\n",
    "        if bidir:\n",
    "            encoder_states = []\n",
    "            layer_inputs = embed_inputs\n",
    "\n",
    "            # build rnn layer-by-layer\n",
    "            for i in range(num_layers):\n",
    "                with tf.variable_scope(\"layer_%d\" % (i + 1)):\n",
    "                    fw_cell = build_rnn_cell(\n",
    "                        1, num_units, cell_type, forget_bias)\n",
    "                    bw_cell = build_rnn_cell(\n",
    "                        1, num_units, cell_type, forget_bias)\n",
    "\n",
    "                    dyn_rnn = tf.nn.bidirectional_dynamic_rnn(\n",
    "                        fw_cell, bw_cell, layer_inputs,\n",
    "                        time_major=time_major,\n",
    "                        dtype=dtype,\n",
    "                        swap_memory=True)\n",
    "                    bi_outputs, (state_fw, state_bw) = dyn_rnn\n",
    "\n",
    "                    # handle cell memory state\n",
    "                    if cell_type == \"LSTM\":\n",
    "                        state_c = state_fw.c + state_bw.c\n",
    "                        state_h = state_fw.h + state_bw.h\n",
    "                        encoder_states.append(LSTMStateTuple(state_c, state_h))\n",
    "                    else:\n",
    "                        encoder_states.append(state_fw + state_bw)\n",
    "\n",
    "                    # concat and map as inputs of next layer\n",
    "                    layer_inputs = tf.layers.dense(\n",
    "                        tf.concat(bi_outputs, -1), num_units)\n",
    "\n",
    "            encoder_outputs = layer_inputs\n",
    "            encoder_states = tuple(encoder_states)\n",
    "\n",
    "        # unidirectional\n",
    "        else:\n",
    "            rnn_cell = build_rnn_cell(\n",
    "                num_layers, num_units, cell_type, forget_bias)\n",
    "\n",
    "            encoder_outputs, encoder_states = tf.nn.dynamic_rnn(\n",
    "                rnn_cell, embed_inputs,\n",
    "                time_major=time_major,\n",
    "                dtype=dtype,\n",
    "                swap_memory=True)\n",
    "\n",
    "    return encoder_outputs, encoder_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = init_embeddings(1000, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "source_ids = tf.placeholder(tf.int32, [None, None])\n",
    "encoder_outputs, encoder_states = build_encoder(embeddings, source_ids, num_layers=2,\n",
    "                                                num_units=256, cell_type=\"LSTM\", bidir=True,\n",
    "                                                name=\"e8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Attention Wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Attention wrapper\n",
    "from tensorflow.contrib.rnn import RNNCell\n",
    "\n",
    "import collections\n",
    "\n",
    "\n",
    "AttnState = collections.namedtuple(\n",
    "    \"AttnState\", (\"cell_states\", \"h\", \"context\"))\n",
    "\n",
    "\n",
    "class AttentionWrapper(RNNCell):\n",
    "    \"\"\"\n",
    "    Attention Wrapper: wrap a rnn_cell with attention mechanism (Bahda 2015)\n",
    "        cell: vanilla multi-layer RNNCell\n",
    "        memory: [batch_size, max_time, num_units]\n",
    "    \"\"\"\n",
    "    def __init__(self, cell, memory, dec_init_states, num_hidden,\n",
    "                 num_units, dtype):\n",
    "        self._cell = cell\n",
    "        self._memory = memory\n",
    "        self.num_hidden = num_hidden\n",
    "\n",
    "        self._dec_init_states = dec_init_states\n",
    "        self._state_size = AttnState(self._cell.state_size,\n",
    "                                     num_units, memory.shape[-1].value)\n",
    "        self._num_units = num_units\n",
    "        self._dtype = dtype\n",
    "\n",
    "    @property\n",
    "    def state_size(self):\n",
    "        return self._state_size\n",
    "\n",
    "    @property\n",
    "    def output_size(self):\n",
    "        return self._num_units\n",
    "\n",
    "    def initial_state(self):\n",
    "        \"\"\"\n",
    "        Generate initial state for attn wrapped rnn cell\n",
    "            dec_init_states: None (no states pass), or encoder final states\n",
    "            num_units: decoder's num of cell units\n",
    "        Returns:\n",
    "            h_0: [batch_size, num_units]\n",
    "            context_0: [batch_size, num_units]\n",
    "        \"\"\"\n",
    "        h_0 = tf.zeros([1, self._num_units], self._dtype)\n",
    "        context_0 = self._compute_context(h_0)\n",
    "        h_0 = context_0 * 0\n",
    "\n",
    "        if self._dec_init_states is None:\n",
    "            batch_size = tf.shape(self._memory)[0]\n",
    "            cell_states = self._cell.zero_state(batch_size, self._dtype)\n",
    "        else:\n",
    "            cell_states = self._dec_init_states\n",
    "\n",
    "        attn_state_0 = AttnState(cell_states, h_0, context_0)\n",
    "\n",
    "        return attn_state_0\n",
    "\n",
    "    def _compute_context(self, query):\n",
    "        \"\"\"\n",
    "        Compute attn scores and weighted sum of memory as the context\n",
    "            query: [batch_size, num_units]\n",
    "        Returns:\n",
    "            context: [batch_size, num_units]\n",
    "        \"\"\"\n",
    "        query = tf.expand_dims(query, -2)\n",
    "        Wq = tf.layers.dense(query, self.num_hidden, use_bias=False)\n",
    "        Wm = tf.layers.dense(self._memory, self.num_hidden, use_bias=False)\n",
    "        e = tf.layers.dense(tf.nn.tanh(Wm + Wq), 1, use_bias=False)\n",
    "        attn_scores = tf.expand_dims(tf.nn.softmax(tf.squeeze(e, axis=-1)), -1)\n",
    "\n",
    "        context = tf.reduce_sum(attn_scores * self._memory, axis=1)\n",
    "\n",
    "        return context\n",
    "\n",
    "    def __call__(self, inputs, attn_states):\n",
    "        \"\"\"\n",
    "            inputs: emebeddings of previous word\n",
    "            states: (cell_states, outputs) at each step\n",
    "        \"\"\"\n",
    "        prev_cell_states, h, context = attn_states\n",
    "\n",
    "        x = tf.concat([inputs, h, context], axis=-1)\n",
    "        new_h, cell_states = self._cell.__call__(x, prev_cell_states)\n",
    "\n",
    "        new_context = self._compute_context(new_h)\n",
    "        new_attn_states = AttnState(cell_states, new_h, new_context)\n",
    "\n",
    "        return (new_h, new_attn_states)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Greedy Decoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Greedy decoding\n",
    "\n",
    "class DecoderOutput(collections.namedtuple(\n",
    "                    \"DecoderOutput\", (\"logits\", \"ids\"))):\n",
    "    \"\"\"\n",
    "        logits: [batch_size, vocab_size]\n",
    "        ids: [batch_size]\n",
    "    \"\"\"\n",
    "    pass\n",
    "\n",
    "\n",
    "class GreedyDecodeCell(object):\n",
    "\n",
    "    def __init__(self, embeddings, cell, dec_init_states,\n",
    "                 output_layer, batch_size, dtype):\n",
    "        self._embeddings = embeddings\n",
    "        self._cell = cell\n",
    "        self._dec_init_states = dec_init_states\n",
    "        self._output_layer = output_layer\n",
    "        self._batch_size = batch_size\n",
    "        self._start_token = tf.nn.embedding_lookup(embeddings, 0)\n",
    "        self._end_id = 1\n",
    "        self._dtype = dtype\n",
    "\n",
    "    @property\n",
    "    def output_dtype(self):\n",
    "        \"\"\"Generate the structure for initial TensorArrays in dynamic_decode\"\"\"\n",
    "        return DecoderOutput(logits=self._dtype, ids=tf.int32)\n",
    "\n",
    "    def initialize(self):\n",
    "        # initial cell states\n",
    "        cell_states = self._dec_init_states\n",
    "\n",
    "        # initial cell inputs: tile [1, embed_dim] => [batch_size, embed_dim]\n",
    "        token = tf.expand_dims(self._start_token, 0)\n",
    "        inputs = tf.tile(token, multiples=[self._batch_size, 1])\n",
    "\n",
    "        # initial ending signals: start with all \"False\"\n",
    "        decode_finished = tf.zeros(shape=[self._batch_size], dtype=tf.bool)\n",
    "\n",
    "        return cell_states, inputs, decode_finished\n",
    "\n",
    "    def step(self, time_index, cell_states, inputs, decode_finished):\n",
    "        # next step of rnn cell and pass the output_layer for logits\n",
    "        new_h, new_cell_states = self._cell.__call__(inputs, cell_states)\n",
    "        logits = self._output_layer(new_h)\n",
    "\n",
    "        # get ids of words predicted and their embeddings\n",
    "        new_ids = tf.cast(tf.argmax(logits, axis=-1), tf.int32)\n",
    "        new_inputs = tf.nn.embedding_lookup(self._embeddings, new_ids)\n",
    "\n",
    "        # make a new output for registering into TensorArrays\n",
    "        new_output = DecoderOutput(logits, new_ids)\n",
    "\n",
    "        # check whether the end_token is reached\n",
    "        new_decode_finished = tf.logical_or(decode_finished,\n",
    "                                            tf.equal(new_ids, self._end_id))\n",
    "\n",
    "        return (new_output, new_cell_states, new_inputs, new_decode_finished)\n",
    "\n",
    "    def finalize(self, final_outputs, final_cell_states):\n",
    "        return final_outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Beam Search Decoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ### Beam Search helpers ###\n",
    "def tile_beam(tensor, beam_size):\n",
    "    \"\"\"\n",
    "        tensor: batch-major, [batch_size, ...]\n",
    "    Returns:\n",
    "        tensor: beam_size tiled, [batch_size, beam_size, ...]\n",
    "    \"\"\"\n",
    "    tensor = tf.expand_dims(tensor, axis=1)\n",
    "    # set multiples: [1, beam_size, 1, ..., 1]\n",
    "    multiples = [1 for i in range(tensor.shape.ndims)]\n",
    "    multiples[1] = beam_size\n",
    "\n",
    "    return tf.tile(tensor, multiples)\n",
    "\n",
    "\n",
    "def merge_batch_beam(tensor):\n",
    "    \"\"\"\n",
    "        tensor: [batch_size, beam_size, ...]\n",
    "    Returns:\n",
    "        tensorL [batch_size * beam_size, ...]\n",
    "    \"\"\"\n",
    "    # tf.shape(t) handles indefinite shape\n",
    "    batch_size = tf.shape(tensor)[0]\n",
    "    # specified shape can be withdrawed right away\n",
    "    beam_size = tensor.shape[1].value\n",
    "\n",
    "    shape = list(tensor.shape)\n",
    "    shape.pop(0)\n",
    "    shape[0] = batch_size * beam_size\n",
    "\n",
    "    return tf.reshape(tensor, shape)\n",
    "\n",
    "\n",
    "def split_batch_beam(tensor, beam_size):\n",
    "    \"\"\"\n",
    "        tensor: [batch_size * beam_size, ...]\n",
    "    Returns:\n",
    "        tensor: [batch_size, beam_size, ...]\n",
    "    \"\"\"\n",
    "    shape = list(tensor.shape)\n",
    "    shape[0] = beam_size\n",
    "    shape.insert(0, -1)\n",
    "\n",
    "    return tf.reshape(tensor, shape)\n",
    "\n",
    "\n",
    "def mask_log_probs(log_probs, end_id, decode_finished):\n",
    "    \"\"\"\n",
    "    Set log_probs after end_token to be [-inf, 0, -inf, ...]\n",
    "        log_probs: [batch_size, beam_size, vocab_size]\n",
    "        decode_finished: [batch_size, beam_size]\n",
    "    \"\"\"\n",
    "    vocab_size = log_probs.shape[-1].value\n",
    "    one_hot = tf.one_hot(end_id, vocab_size, on_value=0.0,\n",
    "                         off_value=log_probs.dtype.min,\n",
    "                         dtype=log_probs.dtype)\n",
    "    I_fin = tf.expand_dims(tf.cast(decode_finished, log_probs.dtype),\n",
    "                           axis=-1)\n",
    "\n",
    "    return (1. - I_fin) * log_probs + I_fin * one_hot\n",
    "\n",
    "\n",
    "def sample_bernoulli(prob, shape):\n",
    "    \"\"\"Samples a boolean tensor with shape = s according to bernouilli\"\"\"\n",
    "    return tf.greater(prob, tf.random_uniform(shape))\n",
    "\n",
    "\n",
    "def add_diversity_penalty(log_probs, div_gamma, div_prob, batch_size,\n",
    "                          beam_size, vocab_size):\n",
    "    \"\"\"\n",
    "    Diversity penalty by Li et al. 2016\n",
    "        div_gamma: (float) diversity parameter\n",
    "        div_prob: adds penalty with div_proba\n",
    "    \"\"\"\n",
    "    if (div_gamma is None) or (div_prob is None):\n",
    "        return log_probs\n",
    "\n",
    "    if (div_gamma == 1) or (div_prob) == 0:\n",
    "        return log_probs\n",
    "\n",
    "    top_probs, top_inds = tf.nn.top_k(log_probs, k=vocab_size, sorted=True)\n",
    "\n",
    "    # inverse permutation to get rank of each entry\n",
    "    top_inds = tf.reshape(top_inds, [-1, vocab_size])\n",
    "    index_rank = tf.map_fn(tf.invert_permutation, top_inds, back_prop=False)\n",
    "    index_rank = tf.reshape(\n",
    "        index_rank, shape=[batch_size, beam_size, vocab_size])\n",
    "\n",
    "    # compute penalty\n",
    "    penalties = tf.log(div_gamma) * tf.cast(index_rank, log_probs.dtype)\n",
    "\n",
    "    # only apply penalty with some probability\n",
    "    apply_penalty = tf.cast(\n",
    "            sample_bernoulli(div_prob, [batch_size, beam_size, vocab_size]),\n",
    "            penalties.dtype)\n",
    "    penalties *= apply_penalty\n",
    "\n",
    "    return log_probs + penalties\n",
    "\n",
    "\n",
    "def gather_helper(tensor, indices, batch_size, beam_size):\n",
    "    \"\"\"\n",
    "        tensor: [batch_size, beam_size, d]\n",
    "        indices: [batch_size, beam_size]\n",
    "    Returns:\n",
    "        new_tensor: new_t[:, i] = t[:, new_parents[:, i]]\n",
    "    \"\"\"\n",
    "    range_ = tf.expand_dims(tf.range(batch_size) * beam_size, axis=1)\n",
    "    # flatten\n",
    "    indices = tf.reshape(indices + range_, [-1])\n",
    "    output = tf.gather(tf.reshape(tensor, [batch_size * beam_size, -1]),\n",
    "                       indices)\n",
    "\n",
    "    if tensor.shape.ndims == 2:\n",
    "        return tf.reshape(output, [batch_size, beam_size])\n",
    "\n",
    "    elif tensor.shape.ndims == 3:\n",
    "        d = tensor.shape[-1].value\n",
    "        return tf.reshape(output, [batch_size, beam_size, d])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Beam search decoding\n",
    "from tensorflow.contrib.framework import nest\n",
    "\n",
    "\n",
    "class BeamDecoderOutput(collections.namedtuple(\n",
    "        \"BeamDecoderOutput\", (\"logits\", \"ids\", \"parents\"))):\n",
    "    \"\"\"\n",
    "        logits: [batch_size, beam_size, vocab_size]\n",
    "        ids: [batch_size, beam_size], best words ids now\n",
    "        parents: [batch_size, beam_size], previous step beam index ids\n",
    "    \"\"\"\n",
    "    pass\n",
    "\n",
    "\n",
    "class BeamDecoderCellStates(collections.namedtuple(\n",
    "        \"BeamDecoderCellStates\", (\"cell_states\", \"log_probs\"))):\n",
    "    \"\"\"\n",
    "        cell_states: [batch_size, beam_size, num_units]\n",
    "        log_probs: [batch_size, beam_size]\n",
    "    \"\"\"\n",
    "    pass\n",
    "\n",
    "\n",
    "class BeamSearchDecodeCell(object):\n",
    "\n",
    "    def __init__(self, embeddings, cell, dec_init_states,\n",
    "                 output_layer, batch_size, dtype, beam_size,\n",
    "                 vocab_size, div_gamma=None, div_prob=None):\n",
    "        \"\"\"\n",
    "            div_gamma: (float) relative weight of penalties\n",
    "            div_prob: (float) prob to apply penalties\n",
    "        \"\"\"\n",
    "        self._embeddings = embeddings\n",
    "        self._vocab_size = vocab_size\n",
    "        self._cell = cell\n",
    "        self._dec_init_states = dec_init_states\n",
    "        self._output_layer = output_layer\n",
    "        self._batch_size = batch_size\n",
    "        self._start_token = tf.nn.embedding_lookup(embeddings, 0)\n",
    "        self._end_id = 1\n",
    "        self._dtype = dtype\n",
    "\n",
    "        self._beam_size = beam_size\n",
    "        self._div_gamma = div_gamma\n",
    "        self._div_prob = div_prob\n",
    "        if hasattr(self._cell, \"_memory\"):\n",
    "            indices = np.repeat(np.arange(self._batch_size), self._beam_size)\n",
    "            self._cell._memory = tf.gather(self._cell._memory, indices)\n",
    "\n",
    "    @property\n",
    "    def output_dtype(self):\n",
    "        \"\"\"Generate the structure for initial TensorArrays in dynamic_decode\"\"\"\n",
    "        return BeamDecoderOutput(logits=self._dtype,\n",
    "                                 ids=tf.int32, parents=tf.int32)\n",
    "\n",
    "    def _initial_state(self):\n",
    "        # t: [batch_size, num_units]\n",
    "        cell_states = nest.map_structure(\n",
    "            lambda t: tile_beam(t, self._beam_size), self._dec_init_states)\n",
    "\n",
    "        # another \"log_probs\" initial states: accumulative log_prob!\n",
    "        log_probs = tf.zeros([self._batch_size, self._beam_size],\n",
    "                             dtype=self._dtype)\n",
    "\n",
    "        return BeamDecoderCellStates(cell_states, log_probs)\n",
    "\n",
    "    def initialize(self):\n",
    "        # initial cell states\n",
    "        cell_states = self._initial_state()\n",
    "\n",
    "        # inputs: SOS, [embed_size] -> [batch_size, beam_size, embed_size]\n",
    "        inputs = tf.tile(tf.reshape(self._start_token, [1, 1, -1]),\n",
    "                         multiples=[self._batch_size, self._beam_size, 1])\n",
    "\n",
    "        # initial ending signals: [batch_size, beam_size]\n",
    "        decode_finished = tf.zeros([self._batch_size, self._beam_size],\n",
    "                                   dtype=tf.bool)\n",
    "\n",
    "        return cell_states, inputs, decode_finished\n",
    "\n",
    "    def step(self, time_index, beam_states, inputs, decode_finished):\n",
    "        \"\"\"\n",
    "            logits: [batch_size, beam_size, vocab_size]\n",
    "            ids: [batch_size, beam_size], best words ids now\n",
    "            parents: [batch_size, beam_size], previous step beam index ids\n",
    "        \"\"\"\n",
    "        # 1-1: merge batch -> [batch_size*beam_size, ...]\n",
    "        cell_states = nest.map_structure(\n",
    "            merge_batch_beam, beam_states.cell_states)\n",
    "        inputs = merge_batch_beam(inputs)\n",
    "\n",
    "        # 1-2: perform cell ops to get new logits\n",
    "        new_h, new_cell_states = self._cell.__call__(inputs, cell_states)\n",
    "        logits = self._output_layer(new_h)\n",
    "\n",
    "        # 1-3: split batch beam -> [batch_size, beam_size, ...]\n",
    "        logits = split_batch_beam(logits, self._beam_size)\n",
    "        new_cell_states = nest.map_structure(\n",
    "            lambda t: split_batch_beam(t, self._beam_size), new_cell_states)\n",
    "\n",
    "        # 2-1: compute log_probs, [batch_size, beam_size, vocab_size]\n",
    "        step_log_probs = tf.nn.log_softmax(logits)\n",
    "        step_log_probs = mask_log_probs(\n",
    "            step_log_probs, self._end_id, decode_finished)\n",
    "\n",
    "        # 2-2: add cumulative log_probs and \"diversity penalty\"\n",
    "        log_probs = tf.expand_dims(beam_states.log_probs, axis=-1)\n",
    "        log_probs = log_probs + step_log_probs\n",
    "        log_probs = add_diversity_penalty(log_probs, self._div_gamma,\n",
    "                                          self._div_prob, self._batch_size,\n",
    "                                          self._beam_size, self._vocab_size)\n",
    "\n",
    "        # 3-1: flatten, if time_index = 0, consider only one beam\n",
    "        # log_probs[:, 0]: [batch_size, vocab_size]\n",
    "        shape = [self._batch_size, self._beam_size * self._vocab_size]\n",
    "        log_probs_flat = tf.reshape(log_probs, shape)\n",
    "        log_probs_flat = tf.cond(time_index > 0, lambda: log_probs_flat,\n",
    "                                 lambda: log_probs[:, 0])\n",
    "\n",
    "        # 3-2: compute the top (beam_size) beams, [batch_size, beam_size]\n",
    "        new_log_probs, indices = tf.nn.top_k(log_probs_flat, self._beam_size)\n",
    "\n",
    "        # 3-3: obtain ids and parent beams\n",
    "        new_ids = indices % self._vocab_size\n",
    "        # //: floor division, know which beam it belongs to\n",
    "        new_parents = indices // self._vocab_size\n",
    "\n",
    "        # 4-1: compute new states\n",
    "        new_inputs = tf.nn.embedding_lookup(self._embeddings, new_ids)\n",
    "\n",
    "        decode_finished = gather_helper(\n",
    "            decode_finished, new_parents, self._batch_size, self._beam_size)\n",
    "\n",
    "        new_decode_finished = tf.logical_or(\n",
    "            decode_finished, tf.equal(new_ids, self._end_id))\n",
    "\n",
    "        new_cell_states = nest.map_structure(\n",
    "            lambda t: gather_helper(t, new_parents, self._batch_size,\n",
    "                                    self._beam_size), new_cell_states)\n",
    "\n",
    "        # 4-2: create new state and output of decoder\n",
    "        new_beam_states = BeamDecoderCellStates(cell_states=new_cell_states,\n",
    "                                                log_probs=new_log_probs)\n",
    "        new_output = BeamDecoderOutput(logits=logits, ids=new_ids,\n",
    "                                       parents=new_parents)\n",
    "\n",
    "        return (new_output, new_beam_states, new_inputs, new_decode_finished)\n",
    "\n",
    "    def finalize(self, final_outputs, final_cell_states):\n",
    "        \"\"\"\n",
    "            final_outputs: [max_time, logits] structure of tensor\n",
    "            final_cell_states: BeamDecoderCellStates\n",
    "        Returns:\n",
    "            [max_time, batch_size, beam_size, ] stucture of tensor\n",
    "        \"\"\"\n",
    "        # reverse the time dimension\n",
    "        max_iter = tf.shape(final_outputs.ids)[0]\n",
    "        final_outputs = nest.map_structure(lambda t: tf.reverse(t, axis=[0]),\n",
    "                                           final_outputs)\n",
    "\n",
    "        # initial states\n",
    "        def create_ta(d):\n",
    "            return tf.TensorArray(dtype=d, size=max_iter)\n",
    "\n",
    "        f_time_index = tf.constant(0, dtype=tf.int32)\n",
    "        # final output dtype\n",
    "        final_dtype = DecoderOutput(logits=self._dtype, ids=tf.int32)\n",
    "        f_output_ta = nest.map_structure(create_ta, final_dtype)\n",
    "\n",
    "        # initial parents: [batch_size, beam_size]\n",
    "        f_parents = tf.tile(\n",
    "            tf.expand_dims(tf.range(self._beam_size), axis=0),\n",
    "            multiples=[self._batch_size, 1])\n",
    "\n",
    "        def condition(f_time_index, output_ta, f_parents):\n",
    "            return tf.less(f_time_index, max_iter)\n",
    "\n",
    "        def body(f_time_index, output_ta, f_parents):\n",
    "            # get ids, logits and parents predicted at this time step\n",
    "            input_t = nest.map_structure(lambda t: t[f_time_index],\n",
    "                                         final_outputs)\n",
    "\n",
    "            # parents: reversed version shows the next position to go\n",
    "            new_beam_state = nest.map_structure(\n",
    "                lambda t: gather_helper(t, f_parents, self._batch_size,\n",
    "                                        self._beam_size),\n",
    "                input_t)\n",
    "\n",
    "            # create new output\n",
    "            new_output = DecoderOutput(logits=new_beam_state.logits,\n",
    "                                       ids=new_beam_state.ids)\n",
    "\n",
    "            # write beam ids\n",
    "            output_ta = nest.map_structure(\n",
    "                lambda ta, out: ta.write(f_time_index, out),\n",
    "                output_ta, new_output)\n",
    "\n",
    "            return (f_time_index + 1), output_ta, input_t.parents\n",
    "\n",
    "        with tf.variable_scope(\"beam_search_decoding\"):\n",
    "            res = tf.while_loop(\n",
    "                    condition,\n",
    "                    body,\n",
    "                    loop_vars=[f_time_index, f_output_ta, f_parents],\n",
    "                    back_prop=False)\n",
    "\n",
    "        # stack the structure and reverse back\n",
    "        final_outputs = nest.map_structure(lambda ta: ta.stack(), res[1])\n",
    "        final_outputs = nest.map_structure(lambda t: tf.reverse(t, axis=[0]),\n",
    "                                           final_outputs)\n",
    "\n",
    "        return DecoderOutput(logits=final_outputs.logits,\n",
    "                             ids=final_outputs.ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Decoding function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dynamic decode function\n",
    "from tensorflow.contrib.framework import nest\n",
    "\n",
    "\n",
    "def transpose_batch_time(tensor):\n",
    "    ndims = tensor.shape.ndims\n",
    "    if ndims == 2:\n",
    "        return tf.transpose(tensor, [1, 0])\n",
    "\n",
    "    elif ndims == 3:\n",
    "        return tf.transpose(tensor, [1, 0, 2])\n",
    "\n",
    "    else:\n",
    "        return tf.transpose(tensor, [1, 0, 2, 3])\n",
    "\n",
    "\n",
    "# Dynamic decode function\n",
    "def dynamic_decode(decoder_cell, max_iter):\n",
    "    max_iter = tf.convert_to_tensor(max_iter, dtype=tf.int32)\n",
    "\n",
    "    # TensorArray: wrap dynamic-sized, per-time-step, write-once Tensor arrays\n",
    "    def create_tensor_array(d):\n",
    "        # initial size = 0\n",
    "        return tf.TensorArray(dtype=d, size=0, dynamic_size=True)\n",
    "\n",
    "    time_index = tf.constant(0, dtype=tf.int32)\n",
    "    # nest.map_structure: applies func to each entry in structure\n",
    "    output_tensor_arrays = nest.map_structure(\n",
    "        create_tensor_array, decoder_cell.output_dtype)\n",
    "\n",
    "    cell_states, inputs, decode_finished = decoder_cell.initialize()\n",
    "\n",
    "    # tf.while_loop(cond, body, vars): Repeat body while condition cond is true\n",
    "    def condition(time_index, output_ta, cell_states, inputs, decode_finished):\n",
    "        \"\"\"\n",
    "            if all \"decode_finished\" are True, return \"False\"\n",
    "        \"\"\"\n",
    "        return tf.logical_not(tf.reduce_all(decode_finished))\n",
    "\n",
    "    def body(time_index, output_ta, cell_states, inputs, decode_finished):\n",
    "        sts = decoder_cell.step(time_index, cell_states, inputs,\n",
    "                                decode_finished)\n",
    "        new_output, new_cell_states, new_inputs, new_decode_finished = sts\n",
    "\n",
    "        # TensorArray.write(index, value): register value and returns new TAs\n",
    "        output_ta = nest.map_structure(\n",
    "            lambda ta, out: ta.write(time_index, out),\n",
    "            output_ta, new_output)\n",
    "\n",
    "        new_decode_finished = tf.logical_or(\n",
    "            tf.greater_equal(time_index, max_iter),\n",
    "            new_decode_finished)\n",
    "\n",
    "        return (time_index + 1, output_ta, new_cell_states, new_inputs,\n",
    "                new_decode_finished)\n",
    "\n",
    "    with tf.variable_scope(\"decoding\"):\n",
    "\n",
    "        res = tf.while_loop(\n",
    "            condition,\n",
    "            body,\n",
    "            loop_vars=[time_index, output_tensor_arrays, cell_states,\n",
    "                       inputs, decode_finished],\n",
    "            back_prop=False)\n",
    "\n",
    "    # get final outputs and states\n",
    "    final_output_ta, final_cell_states = res[1], res[2]\n",
    "\n",
    "    # TA.stack(): stack all tensors in TensorArray, [max_iter+1, batch_size, _]\n",
    "    final_outputs = nest.map_structure(lambda ta: ta.stack(), final_output_ta)\n",
    "\n",
    "    # finalize the computation from the decoder cell\n",
    "    final_outputs = decoder_cell.finalize(final_outputs, final_cell_states)\n",
    "\n",
    "    # transpose the final output\n",
    "    final_outputs = nest.map_structure(transpose_batch_time, final_outputs)\n",
    "\n",
    "    return final_outputs, final_cell_states\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "\n",
    "def build_decoder(encoder_outputs, encoder_states, embeddings,\n",
    "                  num_layers, num_units, cell_type,\n",
    "                  state_pass=True, infer_batch_size=None,\n",
    "                  attention_wrap=None, attn_num_units=128,\n",
    "                  target_ids=None, infer_type=\"greedy\", beam_size=None,\n",
    "                  max_iter=20, dtype=tf.float32, forget_bias=1.0,\n",
    "                  name=\"decoder\"):\n",
    "    \"\"\"\n",
    "    decoder: build rnn decoder with attention and\n",
    "        target_ids: [batch_size, max_time]\n",
    "        infer_type: greedy decode or beam search\n",
    "        attention_wrap: a wrapper to enable attention mechanism\n",
    "\n",
    "    Returns:\n",
    "        train_outputs: logits, [batch_size, max_time, vocab_size]\n",
    "        infer_outputs: namedtuple(logits, ids), [batch_size, max_time, d]\n",
    "    \"\"\"\n",
    "    # parameter checking\n",
    "    if infer_batch_size is None:\n",
    "        txt = \"infer_batch_size not specified, infer output will be 'None'.\"\n",
    "        warnings.warn(txt)\n",
    "    elif infer_type == \"beam_search\" and beam_size is None:\n",
    "        raise ValueError(\"Inference by beam search must specify beam_size.\")\n",
    "\n",
    "    if target_ids is None:\n",
    "        txt = \"target_ids not specified, train_outputs will be 'None'.\"\n",
    "        warnings.warn(txt)\n",
    "\n",
    "    # Build decoder\n",
    "    with tf.variable_scope(name):\n",
    "        vocab_size = embeddings.shape[0].value\n",
    "\n",
    "        # decoder rnn_cell\n",
    "        cell = build_rnn_cell(num_layers, num_units, cell_type, forget_bias)\n",
    "        dec_init_states = encoder_states if state_pass else None\n",
    "        output_layer = tf.layers.Dense(\n",
    "            vocab_size, use_bias=False, name=\"output_projection\")\n",
    "\n",
    "        # wrap with attention\n",
    "        if attention_wrap is not None:\n",
    "            memory = encoder_outputs\n",
    "            cell = attention_wrap(\n",
    "                cell, memory, dec_init_states, attn_num_units,\n",
    "                num_units, dtype)\n",
    "\n",
    "            dec_init_states = cell.initial_state()\n",
    "\n",
    "        # Decode - for training\n",
    "        # pad the token sequences with SOS (Start of Sentence)\n",
    "        train_outputs = None\n",
    "        if target_ids is not None:\n",
    "            input_ids = tf.pad(target_ids, [[0, 0], [1, 0]], constant_values=0)\n",
    "            embed_inputs = tf.nn.embedding_lookup(embeddings, input_ids)\n",
    "\n",
    "            decoder_outputs, decoder_states = tf.nn.dynamic_rnn(\n",
    "                cell, embed_inputs,\n",
    "                initial_state=dec_init_states,\n",
    "                dtype=dtype,\n",
    "                swap_memory=True)\n",
    "\n",
    "            # logits\n",
    "            train_outputs = output_layer(decoder_outputs)\n",
    "\n",
    "        # Decode - for inference\n",
    "        infer_outputs = None\n",
    "        if infer_batch_size is not None:\n",
    "            if dec_init_states is None:\n",
    "                dec_init_states = cell.zero_state(infer_batch_size, dtype)\n",
    "\n",
    "            if infer_type == \"beam_search\":\n",
    "                decoder_cell = BeamSearchDecodeCell(\n",
    "                    embeddings, cell, dec_init_states, output_layer,\n",
    "                    infer_batch_size, dtype, beam_size, vocab_size,\n",
    "                    div_gamma=None, div_prob=None)\n",
    "\n",
    "            else:\n",
    "                decoder_cell = GreedyDecodeCell(\n",
    "                    embeddings, cell, dec_init_states, output_layer,\n",
    "                    infer_batch_size, dtype)\n",
    "\n",
    "            # namedtuple(logits, ids)\n",
    "            infer_outputs, _ = dynamic_decode(decoder_cell, max_iter)\n",
    "\n",
    "    return train_outputs, infer_outputs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aaronlai/env3/lib/python3.5/site-packages/ipykernel_launcher.py:24: UserWarning: infer_batch_size not specified, infer output will be 'None'.\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "target_ids = tf.placeholder(tf.int64, [None, None])\n",
    "logits, infer_outputs = build_decoder(encoder_outputs, encoder_states, embeddings,\n",
    "                       num_layers=2, num_units=256, cell_type=\"LSTM\",\n",
    "                       state_pass=True,\n",
    "                       attention_wrap=AttentionWrapper, attn_num_units=128,\n",
    "                       target_ids=target_ids, name=\"decoder1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0.0009937 , 0.00099725, 0.0010008 , ..., 0.00099409,\n",
       "         0.00099914, 0.000999  ],\n",
       "        [0.00099386, 0.00099876, 0.00100043, ..., 0.00099476,\n",
       "         0.00100046, 0.00099949],\n",
       "        [0.00099505, 0.00100002, 0.00099976, ..., 0.0009953 ,\n",
       "         0.00100057, 0.0009997 ],\n",
       "        [0.00099663, 0.001001  , 0.000999  , ..., 0.00099576,\n",
       "         0.001     , 0.00099963]],\n",
       "\n",
       "       [[0.00099626, 0.00099802, 0.00099754, ..., 0.00099534,\n",
       "         0.00099885, 0.00099671],\n",
       "        [0.00099675, 0.00099832, 0.00099759, ..., 0.00099503,\n",
       "         0.00099836, 0.00099705],\n",
       "        [0.00099709, 0.00099852, 0.00099722, ..., 0.00099454,\n",
       "         0.00099771, 0.00099741],\n",
       "        [0.00099806, 0.00099861, 0.00099653, ..., 0.00099427,\n",
       "         0.0009966 , 0.00099747]]], dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.05)\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=False,\n",
    "                                        gpu_options=gpu_options))\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)\n",
    "results = sess.run(tf.nn.softmax(logits), feed_dict={source_ids: [[3, 3, 3], [4, 5, 6]],\n",
    "                                                     target_ids: [[8, 8, 8], [8, 10, 12]]})\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aaronlai/env3/lib/python3.5/site-packages/ipykernel_launcher.py:30: UserWarning: target_ids not specified, train_outputs will be 'None'.\n"
     ]
    }
   ],
   "source": [
    "# inference\n",
    "logits, infer_outputs = build_decoder(encoder_outputs, encoder_states, embeddings,\n",
    "                        num_layers=2, num_units=256, cell_type=\"LSTM\",\n",
    "                        state_pass=False, infer_batch_size=3,\n",
    "                        attention_wrap=AttentionWrapper, attn_num_units=128,\n",
    "                        infer_type=\"beam_search\", beam_size=5,\n",
    "                        max_iter=10, name=\"a3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecoderOutput(logits=array([[[[ 2.0948760e-04,  2.8687969e-04,  1.8958990e-03, ...,\n",
       "           7.8416977e-04, -1.4875550e-04, -3.4679662e-04],\n",
       "         [ 2.0948760e-04,  2.8687969e-04,  1.8958990e-03, ...,\n",
       "           7.8416977e-04, -1.4875550e-04, -3.4679662e-04],\n",
       "         [ 2.0948760e-04,  2.8687969e-04,  1.8958990e-03, ...,\n",
       "           7.8416977e-04, -1.4875550e-04, -3.4679662e-04],\n",
       "         [ 2.0948760e-04,  2.8687969e-04,  1.8958990e-03, ...,\n",
       "           7.8416977e-04, -1.4875550e-04, -3.4679662e-04],\n",
       "         [ 2.0948760e-04,  2.8687969e-04,  1.8958990e-03, ...,\n",
       "           7.8416977e-04, -1.4875550e-04, -3.4679662e-04]],\n",
       "\n",
       "        [[ 3.7697289e-04,  9.5821812e-04,  2.9059462e-03, ...,\n",
       "           7.7804015e-04, -4.6190570e-04,  4.2391484e-04],\n",
       "         [ 3.7697289e-04,  9.5821812e-04,  2.9059462e-03, ...,\n",
       "           7.7804015e-04, -4.6190570e-04,  4.2391484e-04],\n",
       "         [ 3.7697289e-04,  9.5821812e-04,  2.9059462e-03, ...,\n",
       "           7.7804015e-04, -4.6190570e-04,  4.2391484e-04],\n",
       "         [ 6.1858364e-04,  4.6400641e-04,  2.8180864e-03, ...,\n",
       "           2.0412304e-03,  3.4051272e-04, -7.8794372e-04],\n",
       "         [ 8.7882415e-04,  5.5620132e-04,  3.0712830e-03, ...,\n",
       "           1.0485391e-03, -4.3751841e-04, -5.3147331e-04]],\n",
       "\n",
       "        [[ 1.1862960e-03,  1.2411298e-03,  3.6001126e-03, ...,\n",
       "           5.9698103e-04, -8.6264935e-04,  2.6373682e-04],\n",
       "         [ 1.0936174e-03,  1.3886640e-03,  3.4125177e-03, ...,\n",
       "           6.4945233e-04, -7.6075608e-04,  5.8547599e-04],\n",
       "         [ 1.0936174e-03,  1.3886640e-03,  3.4125177e-03, ...,\n",
       "           6.4945233e-04, -7.6075608e-04,  5.8547599e-04],\n",
       "         [ 1.0936174e-03,  1.3886640e-03,  3.4125177e-03, ...,\n",
       "           6.4945233e-04, -7.6075608e-04,  5.8547599e-04],\n",
       "         [ 5.9184327e-04,  1.7975259e-03,  3.2458277e-03, ...,\n",
       "           3.7177629e-04, -7.9009810e-04,  1.5468342e-03]],\n",
       "\n",
       "        ...,\n",
       "\n",
       "        [[ 5.1626018e-03,  5.4905694e-03,  5.8919890e-03, ...,\n",
       "          -1.9432394e-03, -1.6891060e-03,  1.4673281e-04],\n",
       "         [ 5.1626018e-03,  5.4905694e-03,  5.8919890e-03, ...,\n",
       "          -1.9432394e-03, -1.6891060e-03,  1.4673281e-04],\n",
       "         [ 5.1626018e-03,  5.4905694e-03,  5.8919890e-03, ...,\n",
       "          -1.9432394e-03, -1.6891060e-03,  1.4673281e-04],\n",
       "         [ 5.1626018e-03,  5.4905694e-03,  5.8919890e-03, ...,\n",
       "          -1.9432394e-03, -1.6891060e-03,  1.4673281e-04],\n",
       "         [ 5.1626018e-03,  5.4905694e-03,  5.8919890e-03, ...,\n",
       "          -1.9432394e-03, -1.6891060e-03,  1.4673281e-04]],\n",
       "\n",
       "        [[ 5.4652086e-03,  5.5308058e-03,  4.9921824e-03, ...,\n",
       "          -1.9053982e-03, -1.8420622e-03,  4.5611261e-05],\n",
       "         [ 5.2970005e-03,  6.1417026e-03,  6.3023013e-03, ...,\n",
       "          -2.0704311e-03, -1.6255049e-03,  4.3229508e-04],\n",
       "         [ 5.4652086e-03,  5.5308058e-03,  4.9921824e-03, ...,\n",
       "          -1.9053982e-03, -1.8420622e-03,  4.5611261e-05],\n",
       "         [ 5.4652086e-03,  5.5308058e-03,  4.9921824e-03, ...,\n",
       "          -1.9053982e-03, -1.8420622e-03,  4.5611261e-05],\n",
       "         [ 4.7979616e-03,  5.1488746e-03,  5.3449208e-03, ...,\n",
       "          -1.8800376e-03, -1.7995453e-03, -1.5523168e-04]],\n",
       "\n",
       "        [[ 4.7931741e-03,  5.6627542e-03,  5.6256810e-03, ...,\n",
       "          -1.9556042e-03, -1.7193295e-03,  1.7493666e-04],\n",
       "         [ 5.2928636e-03,  6.6544982e-03,  6.5818457e-03, ...,\n",
       "          -2.1468657e-03, -1.5475244e-03,  7.6198101e-04],\n",
       "         [ 4.9166842e-03,  5.5976701e-03,  6.1082290e-03, ...,\n",
       "          -2.0711205e-03, -1.6673467e-03,  8.1583078e-04],\n",
       "         [ 5.4610074e-03,  6.0447729e-03,  5.2734455e-03, ...,\n",
       "          -1.9792775e-03, -1.7628516e-03,  3.7427229e-04],\n",
       "         [ 5.0275419e-03,  6.3858894e-03,  6.1067333e-03, ...,\n",
       "          -2.0430002e-03, -2.1087851e-03,  1.0232603e-03]]],\n",
       "\n",
       "\n",
       "       [[[ 1.9279898e-04,  2.4942227e-04,  1.8924843e-03, ...,\n",
       "           7.8830431e-04, -1.6538611e-04, -4.0907133e-04],\n",
       "         [ 1.9279898e-04,  2.4942227e-04,  1.8924843e-03, ...,\n",
       "           7.8830431e-04, -1.6538611e-04, -4.0907133e-04],\n",
       "         [ 1.9279898e-04,  2.4942227e-04,  1.8924843e-03, ...,\n",
       "           7.8830431e-04, -1.6538611e-04, -4.0907133e-04],\n",
       "         [ 1.9279898e-04,  2.4942227e-04,  1.8924843e-03, ...,\n",
       "           7.8830431e-04, -1.6538611e-04, -4.0907133e-04],\n",
       "         [ 1.9279898e-04,  2.4942227e-04,  1.8924843e-03, ...,\n",
       "           7.8830431e-04, -1.6538611e-04, -4.0907133e-04]],\n",
       "\n",
       "        [[ 3.3603804e-04,  8.6396339e-04,  2.8919545e-03, ...,\n",
       "           7.9747121e-04, -5.0610612e-04,  2.6481418e-04],\n",
       "         [ 3.3603804e-04,  8.6396339e-04,  2.8919545e-03, ...,\n",
       "           7.9747121e-04, -5.0610612e-04,  2.6481418e-04],\n",
       "         [ 5.7773653e-04,  3.6974798e-04,  2.8042088e-03, ...,\n",
       "           2.0589295e-03,  2.9678695e-04, -9.4866037e-04],\n",
       "         [ 3.3603804e-04,  8.6396339e-04,  2.8919545e-03, ...,\n",
       "           7.9747121e-04, -5.0610612e-04,  2.6481418e-04],\n",
       "         [ 8.3820551e-04,  4.6099175e-04,  3.0576466e-03, ...,\n",
       "           1.0675332e-03, -4.8094330e-04, -6.9241109e-04]],\n",
       "\n",
       "        [[ 1.1190162e-03,  1.0777798e-03,  3.5709343e-03, ...,\n",
       "           6.4227055e-04, -9.3867735e-04, -9.6804288e-06],\n",
       "         [ 1.0260893e-03,  1.2246210e-03,  3.3822854e-03, ...,\n",
       "           6.9417420e-04, -8.3699985e-04,  3.1207586e-04],\n",
       "         [ 1.1190162e-03,  1.0777798e-03,  3.5709343e-03, ...,\n",
       "           6.4227055e-04, -9.3867735e-04, -9.6804288e-06],\n",
       "         [ 3.8235245e-04,  1.4474774e-03,  3.3389144e-03, ...,\n",
       "           9.9913985e-04, -7.6740549e-04,  3.1434122e-04],\n",
       "         [ 1.0260893e-03,  1.2246210e-03,  3.3822854e-03, ...,\n",
       "           6.9417420e-04, -8.3699985e-04,  3.1207586e-04]],\n",
       "\n",
       "        ...,\n",
       "\n",
       "        [[ 4.4441959e-03,  3.1416181e-03,  6.2799861e-04, ...,\n",
       "          -3.3988762e-03, -1.6005813e-03, -1.3865433e-03],\n",
       "         [ 4.9931915e-03,  3.7164844e-03, -1.0215149e-03, ...,\n",
       "          -3.6060854e-03, -5.5182609e-04, -1.8129157e-03],\n",
       "         [ 4.4441959e-03,  3.1416181e-03,  6.2799861e-04, ...,\n",
       "          -3.3988762e-03, -1.6005813e-03, -1.3865433e-03],\n",
       "         [ 5.3322180e-03,  3.0829140e-03, -1.9541263e-04, ...,\n",
       "          -3.8396404e-03, -9.6478977e-04, -3.0178272e-03],\n",
       "         [ 4.4441959e-03,  3.1416181e-03,  6.2799861e-04, ...,\n",
       "          -3.3988762e-03, -1.6005813e-03, -1.3865433e-03]],\n",
       "\n",
       "        [[ 5.2359938e-03,  3.5477658e-03, -1.1064513e-03, ...,\n",
       "          -4.4079963e-03, -8.4451219e-04, -2.6567033e-03],\n",
       "         [ 5.2359938e-03,  3.5477658e-03, -1.1064513e-03, ...,\n",
       "          -4.4079963e-03, -8.4451219e-04, -2.6567033e-03],\n",
       "         [ 5.1512830e-03,  3.3721086e-03, -1.1192460e-04, ...,\n",
       "          -5.0730081e-03, -1.3017494e-03, -3.1896837e-03],\n",
       "         [ 5.2359938e-03,  3.5477658e-03, -1.1064513e-03, ...,\n",
       "          -4.4079963e-03, -8.4451219e-04, -2.6567033e-03],\n",
       "         [ 5.7849316e-03,  3.8743175e-03, -1.8857315e-03, ...,\n",
       "          -5.1505286e-03, -2.0681770e-04, -3.6739162e-03]],\n",
       "\n",
       "        [[ 5.8890134e-03,  3.9245589e-03, -2.8391164e-03, ...,\n",
       "          -5.3927521e-03, -7.9289312e-05, -3.8229045e-03],\n",
       "         [ 5.6573646e-03,  3.9034544e-03, -1.7869172e-03, ...,\n",
       "          -6.1989073e-03, -5.4691592e-04, -4.1302061e-03],\n",
       "         [ 5.3730048e-03,  2.8398794e-03, -2.2178541e-03, ...,\n",
       "          -4.9335239e-03, -6.1790925e-05, -4.0392345e-03],\n",
       "         [ 6.5627396e-03,  4.2794561e-03, -4.5634173e-03, ...,\n",
       "          -5.2887313e-03,  9.2395709e-04, -4.3209950e-03],\n",
       "         [ 5.2712290e-03,  3.4659638e-03, -1.9104169e-03, ...,\n",
       "          -4.9589821e-03, -4.9516791e-04, -3.5620972e-03]]],\n",
       "\n",
       "\n",
       "       [[[ 2.3992769e-04,  2.1131686e-04,  1.8891578e-03, ...,\n",
       "           8.3194446e-04, -1.2914262e-04, -3.7224751e-04],\n",
       "         [ 2.3992769e-04,  2.1131686e-04,  1.8891578e-03, ...,\n",
       "           8.3194446e-04, -1.2914262e-04, -3.7224751e-04],\n",
       "         [ 2.3992769e-04,  2.1131686e-04,  1.8891578e-03, ...,\n",
       "           8.3194446e-04, -1.2914262e-04, -3.7224751e-04],\n",
       "         [ 2.3992769e-04,  2.1131686e-04,  1.8891578e-03, ...,\n",
       "           8.3194446e-04, -1.2914262e-04, -3.7224751e-04],\n",
       "         [ 2.3992769e-04,  2.1131686e-04,  1.8891578e-03, ...,\n",
       "           8.3194446e-04, -1.2914262e-04, -3.7224751e-04]],\n",
       "\n",
       "        [[ 4.4089864e-04,  7.7597360e-04,  2.8828848e-03, ...,\n",
       "           8.9760643e-04, -4.0982198e-04,  3.5395089e-04],\n",
       "         [ 4.4089864e-04,  7.7597360e-04,  2.8828848e-03, ...,\n",
       "           8.9760643e-04, -4.0982198e-04,  3.5395089e-04],\n",
       "         [ 4.4089864e-04,  7.7597360e-04,  2.8828848e-03, ...,\n",
       "           8.9760643e-04, -4.0982198e-04,  3.5395089e-04],\n",
       "         [ 6.8231730e-04,  2.8071681e-04,  2.7937917e-03, ...,\n",
       "           2.1594986e-03,  3.9247583e-04, -8.5929898e-04],\n",
       "         [ 5.0633692e-04,  6.1831262e-04,  2.9199170e-03, ...,\n",
       "           1.3818461e-03, -2.5464679e-04, -4.1935779e-04]],\n",
       "\n",
       "        [[ 1.2755605e-03,  9.4121648e-04,  3.5499402e-03, ...,\n",
       "           7.9684006e-04, -7.7136455e-04,  1.3212649e-04],\n",
       "         [ 1.5263546e-03,  4.4285390e-04,  3.4617477e-03, ...,\n",
       "           2.0725434e-03,  3.1727715e-05, -1.0890366e-03],\n",
       "         [ 1.2755605e-03,  9.4121648e-04,  3.5499402e-03, ...,\n",
       "           7.9684006e-04, -7.7136455e-04,  1.3212649e-04],\n",
       "         [ 5.3797348e-04,  1.3105264e-03,  3.3204290e-03, ...,\n",
       "           1.1533899e-03, -6.0022902e-04,  4.5597646e-04],\n",
       "         [ 1.5263546e-03,  4.4285390e-04,  3.4617477e-03, ...,\n",
       "           2.0725434e-03,  3.1727715e-05, -1.0890366e-03]],\n",
       "\n",
       "        ...,\n",
       "\n",
       "        [[ 4.6976777e-03,  2.8903282e-03,  4.9451191e-04, ...,\n",
       "          -3.1418647e-03, -1.0510284e-03, -1.1206506e-03],\n",
       "         [ 4.7985297e-03,  2.5528220e-03,  1.3787319e-03, ...,\n",
       "          -3.6052312e-03, -1.4618621e-03, -1.9144302e-03],\n",
       "         [ 4.6976777e-03,  2.8903282e-03,  4.9451191e-04, ...,\n",
       "          -3.1418647e-03, -1.0510284e-03, -1.1206506e-03],\n",
       "         [ 4.6976777e-03,  2.8903282e-03,  4.9451191e-04, ...,\n",
       "          -3.1418647e-03, -1.0510284e-03, -1.1206506e-03],\n",
       "         [ 4.7985297e-03,  2.5528220e-03,  1.3787319e-03, ...,\n",
       "          -3.6052312e-03, -1.4618621e-03, -1.9144302e-03]],\n",
       "\n",
       "        [[ 5.4934118e-03,  3.3019714e-03, -1.2630369e-03, ...,\n",
       "          -4.1546212e-03, -2.5682565e-04, -2.4003109e-03],\n",
       "         [ 5.4934118e-03,  3.3019714e-03, -1.2630369e-03, ...,\n",
       "          -4.1546212e-03, -2.5682565e-04, -2.4003109e-03],\n",
       "         [ 6.1307242e-03,  3.8046332e-03, -3.0327719e-03, ...,\n",
       "          -4.2332616e-03,  8.3712087e-04, -2.8806559e-03],\n",
       "         [ 5.4128217e-03,  3.1243134e-03, -2.7546193e-04, ...,\n",
       "          -4.8159650e-03, -7.1400765e-04, -2.9357409e-03],\n",
       "         [ 5.4934118e-03,  3.3019714e-03, -1.2630369e-03, ...,\n",
       "          -4.1546212e-03, -2.5682565e-04, -2.4003109e-03]],\n",
       "\n",
       "        [[ 6.1504841e-03,  3.6882742e-03, -3.0175871e-03, ...,\n",
       "          -5.1441267e-03,  5.3940580e-04, -3.5789302e-03],\n",
       "         [ 5.9233168e-03,  3.6649951e-03, -1.9718413e-03, ...,\n",
       "          -5.9471447e-03,  7.1572140e-05, -3.8884906e-03],\n",
       "         [ 5.4803859e-03,  3.6122845e-03, -2.2278517e-03, ...,\n",
       "          -4.7877315e-03,  1.3699412e-04, -2.9099681e-03],\n",
       "         [ 5.5316798e-03,  3.2265156e-03, -2.0897775e-03, ...,\n",
       "          -4.7107581e-03,  1.2356625e-04, -3.3172625e-03],\n",
       "         [ 5.3088772e-03,  3.2041355e-03, -1.0460619e-03, ...,\n",
       "          -5.5122795e-03, -3.4250505e-04, -3.6282297e-03]]]],\n",
       "      dtype=float32), ids=array([[[828, 828,  65,   3,  65],\n",
       "        [ 65,  65,  65, 138, 828],\n",
       "        [ 65, 828, 828, 828, 828],\n",
       "        [828, 828, 828,  65, 828],\n",
       "        [622, 828,  65, 622, 828],\n",
       "        [622,  65, 622, 622, 828],\n",
       "        [622, 622,  65, 622, 622],\n",
       "        [622, 622, 622, 622, 622],\n",
       "        [622, 622, 622, 622, 622],\n",
       "        [622, 478, 622, 622, 119],\n",
       "        [478, 478, 101, 735, 478]],\n",
       "\n",
       "       [[828, 828,  65,  65,   3],\n",
       "        [ 65,  65, 138,  65,  65],\n",
       "        [ 65, 828,  65, 828, 828],\n",
       "        [828,  65,  65,  65, 828],\n",
       "        [828, 828,  65, 828, 828],\n",
       "        [828, 828, 828, 828, 828],\n",
       "        [828, 268, 268, 828, 268],\n",
       "        [268, 268, 268, 268, 268],\n",
       "        [268, 268, 268, 268, 268],\n",
       "        [268, 268, 268, 268, 945],\n",
       "        [268, 301, 131, 447, 268]],\n",
       "\n",
       "       [[828, 828,  65,  65,   3],\n",
       "        [ 65,  65,  65, 138,  65],\n",
       "        [ 65, 828,  65, 828, 828],\n",
       "        [828, 828,  65,  65, 828],\n",
       "        [828, 828,  65, 828,  65],\n",
       "        [828, 828, 828, 828, 828],\n",
       "        [828, 828, 268, 268, 268],\n",
       "        [268, 268, 268, 268, 268],\n",
       "        [268, 268, 268, 268, 268],\n",
       "        [268, 268, 945, 268, 268],\n",
       "        [268, 301, 268, 268, 945]]], dtype=int32))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)\n",
    "inf_results = sess.run(infer_outputs, feed_dict={source_ids: [[3, 3, 3], [4, 5, 6], [10, 10, 10]]})\n",
    "inf_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[828, 828,  65,   3,  65],\n",
       "        [ 65,  65,  65, 138, 828],\n",
       "        [ 65, 828, 828, 828, 828],\n",
       "        [828, 828, 828,  65, 828],\n",
       "        [622, 828,  65, 622, 828],\n",
       "        [622,  65, 622, 622, 828],\n",
       "        [622, 622,  65, 622, 622],\n",
       "        [622, 622, 622, 622, 622],\n",
       "        [622, 622, 622, 622, 622],\n",
       "        [622, 478, 622, 622, 119],\n",
       "        [478, 478, 101, 735, 478]],\n",
       "\n",
       "       [[828, 828,  65,  65,   3],\n",
       "        [ 65,  65, 138,  65,  65],\n",
       "        [ 65, 828,  65, 828, 828],\n",
       "        [828,  65,  65,  65, 828],\n",
       "        [828, 828,  65, 828, 828],\n",
       "        [828, 828, 828, 828, 828],\n",
       "        [828, 268, 268, 828, 268],\n",
       "        [268, 268, 268, 268, 268],\n",
       "        [268, 268, 268, 268, 268],\n",
       "        [268, 268, 268, 268, 945],\n",
       "        [268, 301, 131, 447, 268]],\n",
       "\n",
       "       [[828, 828,  65,  65,   3],\n",
       "        [ 65,  65,  65, 138,  65],\n",
       "        [ 65, 828,  65, 828, 828],\n",
       "        [828, 828,  65,  65, 828],\n",
       "        [828, 828,  65, 828,  65],\n",
       "        [828, 828, 828, 828, 828],\n",
       "        [828, 828, 268, 268, 268],\n",
       "        [268, 268, 268, 268, 268],\n",
       "        [268, 268, 268, 268, 268],\n",
       "        [268, 268, 945, 268, 268],\n",
       "        [268, 301, 268, 268, 945]]], dtype=int32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inf_results.ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training and Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute loss\n",
    "def compute_loss(source_ids, target_ids, sequence_mask, embeddings,\n",
    "                 enc_num_layers, enc_num_units, enc_cell_type, enc_bidir,\n",
    "                 dec_num_layers, dec_num_units, dec_cell_type, state_pass,\n",
    "                 infer_batch_size, infer_type=\"greedy\", beam_size=None, max_iter=20,\n",
    "                 attn_wrapper=None, attn_num_units=128, l2_regularize=None,\n",
    "                 name=\"Seq2seq\"):\n",
    "    \"\"\"\n",
    "    Creates a Seq2seq model and returns cross entropy loss.\n",
    "    \"\"\"\n",
    "    with tf.name_scope(name):\n",
    "        # build encoder\n",
    "        encoder_outputs, encoder_states = build_encoder(\n",
    "            embeddings, source_ids, enc_num_layers, enc_num_units,\n",
    "            enc_cell_type, bidir=enc_bidir, name=\"%s_encoder\" % name)\n",
    "\n",
    "        # build decoder: logits, [batch_size, max_time, vocab_size]\n",
    "        train_logits, infer_outputs = build_decoder(\n",
    "            encoder_outputs, encoder_states, embeddings,\n",
    "            dec_num_layers, dec_num_units, dec_cell_type,\n",
    "            state_pass, infer_batch_size, attn_wrapper, attn_num_units,\n",
    "            target_ids, infer_type, beam_size, max_iter,\n",
    "            name=\"%s_decoder\" % name)\n",
    "\n",
    "        # compute loss\n",
    "        with tf.name_scope('loss'):\n",
    "            final_ids = tf.pad(target_ids, [[0, 0], [0, 1]], constant_values=1)\n",
    "            losses = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                logits=train_logits, labels=final_ids)\n",
    "\n",
    "            losses = tf.boolean_mask(losses[:, :-1], sequence_mask)\n",
    "            reduced_loss = tf.reduce_mean(losses)\n",
    "            CE = tf.reduce_sum(losses)\n",
    "\n",
    "            if l2_regularize is None:\n",
    "                return CE, reduced_loss, train_logits, infer_outputs\n",
    "            else:\n",
    "                l2_loss = tf.add_n([tf.nn.l2_loss(v)\n",
    "                                    for v in tf.trainable_variables()\n",
    "                                    if not('bias' in v.name)])\n",
    "\n",
    "                total_loss = reduced_loss + l2_regularize * l2_loss\n",
    "                return CE, total_loss, train_logits, infer_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_perplexity(sess, CE, mask, feed_dict):\n",
    "    \"\"\"\n",
    "    Compute perplexity for a batch of data\n",
    "    \"\"\"\n",
    "    CE_words = sess.run(CE, feed_dict=feed_dict)\n",
    "    N_words = np.sum(mask)\n",
    "    return np.exp(CE_words / N_words)\n",
    "\n",
    "\n",
    "def loadfile(filename, is_source, max_length):\n",
    "    \"\"\"\n",
    "    Load and clean data\n",
    "    \"\"\"\n",
    "    def clean(row):\n",
    "        row = np.array(row.split(), dtype=np.int32)\n",
    "        leng = len(row)\n",
    "        if leng < max_length:\n",
    "            if is_source:\n",
    "                # represents constant zero padding\n",
    "                pads = -np.ones(max_length - leng, dtype=np.int32)\n",
    "                row = np.concatenate((pads, row))\n",
    "            else:\n",
    "                # represents EOS token\n",
    "                pads = -2 * np.ones(max_length - leng, dtype=np.int32)\n",
    "                row = np.concatenate((row, pads))\n",
    "        elif leng > max_length:\n",
    "            row = row[:max_length]\n",
    "        return row\n",
    "\n",
    "    df = pd.read_csv(filename, header=None, index_col=None)\n",
    "    data = np.array(df[0].apply(lambda t: clean(t)).tolist(), dtype=np.int32)\n",
    "    return data\n",
    "\n",
    "# saving and load\n",
    "def load(saver, sess, logdir):\n",
    "    print(\"Trying to restore saved checkpoints from {} ...\".format(logdir),\n",
    "          end=\"\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(logdir)\n",
    "    if ckpt:\n",
    "        print(\"  Checkpoint found: {}\".format(ckpt.model_checkpoint_path))\n",
    "        global_step = int(ckpt.model_checkpoint_path\n",
    "                          .split('/')[-1]\n",
    "                          .split('-')[-1])\n",
    "        print(\"  Global step was: {}\".format(global_step))\n",
    "        print(\"  Restoring...\", end=\"\")\n",
    "        saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "        print(\" Done.\")\n",
    "        return global_step\n",
    "    else:\n",
    "        print(\" No checkpoint found.\")\n",
    "        return None\n",
    "\n",
    "\n",
    "def save(saver, sess, logdir, step):\n",
    "    model_name = 'model.ckpt'\n",
    "    checkpoint_path = os.path.join(logdir, model_name)\n",
    "    print('Storing checkpoint to {} ...'.format(logdir), end=\"\")\n",
    "    sys.stdout.flush()\n",
    "\n",
    "    if not os.path.exists(logdir):\n",
    "        os.makedirs(logdir)\n",
    "\n",
    "    saver.save(sess, checkpoint_path, global_step=step)\n",
    "    print(' Done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# YAML configuration\n",
    "def get_model_config(config):\n",
    "    enc_num_layers = config[\"encoder\"][\"num_layers\"]\n",
    "    enc_num_units = config[\"encoder\"][\"num_units\"]\n",
    "    enc_cell_type = config[\"encoder\"][\"cell_type\"]\n",
    "    enc_bidir = config[\"encoder\"][\"bidirectional\"]\n",
    "    dec_num_layers = config[\"decoder\"][\"num_layers\"]\n",
    "    dec_num_units = config[\"decoder\"][\"num_units\"]\n",
    "    dec_cell_type = config[\"decoder\"][\"cell_type\"]\n",
    "    state_pass = config[\"decoder\"][\"state_pass\"]\n",
    "    infer_batch_size = config[\"inference\"][\"infer_batch_size\"]\n",
    "    infer_type = config[\"inference\"][\"type\"]\n",
    "    beam_size = config[\"inference\"][\"beam_size\"]\n",
    "    max_iter = config[\"inference\"][\"max_length\"]\n",
    "    attn_num_units = config[\"decoder\"][\"attn_num_units\"]\n",
    "    l2_regularize = config[\"training\"][\"l2_regularize\"]\n",
    "\n",
    "    return (enc_num_layers, enc_num_units, enc_cell_type, enc_bidir,\n",
    "            dec_num_layers, dec_num_units, dec_cell_type, state_pass,\n",
    "            infer_batch_size, infer_type, beam_size, max_iter,\n",
    "            attn_num_units, l2_regularize)\n",
    "\n",
    "\n",
    "def get_training_config(config):\n",
    "    train_config = config[\"training\"]\n",
    "    logdir = train_config[\"logdir\"]\n",
    "    restore_from = train_config[\"restore_from\"]\n",
    "\n",
    "    learning_rate = train_config[\"learning_rate\"]\n",
    "    gpu_fraction = train_config[\"gpu_fraction\"]\n",
    "    max_checkpoints = train_config[\"max_checkpoints\"]\n",
    "    train_steps = train_config[\"train_steps\"]\n",
    "    batch_size = train_config[\"batch_size\"]\n",
    "    print_every = train_config[\"print_every\"]\n",
    "    checkpoint_every = train_config[\"checkpoint_every\"]\n",
    "\n",
    "    s_filename = train_config[\"train_source_file\"]\n",
    "    t_filename = train_config[\"train_target_file\"]\n",
    "    s_max_leng = train_config[\"source_max_length\"]\n",
    "    t_max_leng = train_config[\"target_max_length\"]\n",
    "\n",
    "    dev_s_filename = train_config[\"dev_source_file\"]\n",
    "    dev_t_filename = train_config[\"dev_target_file\"]\n",
    "\n",
    "    loss_fig = train_config[\"loss_fig\"]\n",
    "    perp_fig = train_config[\"perplexity_fig\"]\n",
    "\n",
    "    return (logdir, restore_from, learning_rate, gpu_fraction, max_checkpoints,\n",
    "            train_steps, batch_size, print_every, checkpoint_every,\n",
    "            s_filename, t_filename, s_max_leng, t_max_leng, dev_s_filename,\n",
    "            dev_t_filename, loss_fig, perp_fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "\n",
    "config_details = {\n",
    "    \"Name\": \"Seq2seq_BiLSTM_Attn_BeamSearch\",\n",
    "    \"embeddings\": {\n",
    "        \"vocab_size\": 1000,\n",
    "        \"embed_size\": 128,\n",
    "    },\n",
    "    \"encoder\": {\n",
    "        \"num_layers\": 2,\n",
    "        \"num_units\": 256,\n",
    "        \"cell_type\": \"LSTM\",\n",
    "        \"bidirectional\": True,\n",
    "    },\n",
    "    \"decoder\": {\n",
    "        \"num_layers\": 2,\n",
    "        \"num_units\": 256,\n",
    "        \"cell_type\": \"LSTM\",\n",
    "        \"state_pass\": True,\n",
    "        \"wrapper\": \"Attention\",\n",
    "        \"attn_num_units\": 128,\n",
    "    },\n",
    "    \"inference\": {\n",
    "        \"infer_batch_size\": 15,\n",
    "        \"type\": \"beam_search\",\n",
    "        \"beam_size\": 5,\n",
    "        \"max_length\": 20,\n",
    "        \"infer_source_file\": \"./example/dev_source.txt\",\n",
    "        \"infer_source_max_length\": 25,\n",
    "        \"output_path\": \"./prediction.txt\",\n",
    "    },\n",
    "    \"training\": {\n",
    "        \"l2_regularize\": None,\n",
    "        \"logdir\": \"./log_s2sattn/\",\n",
    "        \"restore_from\": \"./log_s2sattn/\",\n",
    "        \"learning_rate\": 1e-3,\n",
    "        \"batch_size\": 64,\n",
    "        \"gpu_fraction\": 0.05,\n",
    "        \"max_checkpoints\": 10000,\n",
    "        \"train_steps\": 5000,\n",
    "        \"print_every\": 20,\n",
    "        \"checkpoint_every\": 1000,\n",
    "        \"train_source_file\": \"./example/train_source.txt\",\n",
    "        \"train_target_file\": \"./example/train_target.txt\",\n",
    "        \"dev_source_file\": \"./example/dev_source.txt\",\n",
    "        \"dev_target_file\": \"./example/dev_target.txt\",\n",
    "        \"source_max_length\": 25,\n",
    "        \"target_max_length\": 25,\n",
    "        \"loss_fig\": \"./training_loss_over_time\",\n",
    "        \"perplexity_fig\": \"./perplexity_over_time\",\n",
    "    }\n",
    "}\n",
    "\n",
    "with open('./configs/config_seq2seqAttn_beamsearch.yaml', \"w\") as f:\n",
    "    yaml.dump({\"configuration\": config_details}, f, default_flow_style=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open('./configs/config_seq2seqAttn_beamsearch.yaml') as f:\n",
    "    # use safe_load instead load\n",
    "    config = yaml.safe_load(f)\n",
    "\n",
    "config = config[\"configuration\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Construct model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing embeddings ...\n",
      "\tDone.\n",
      "Building model architecture ...\n",
      "\tDone.\n"
     ]
    }
   ],
   "source": [
    "# loading configurations\n",
    "name = config[\"Name\"]\n",
    "\n",
    "# Construct or load embeddings\n",
    "print(\"Initializing embeddings ...\")\n",
    "vocab_size = config[\"embeddings\"][\"vocab_size\"]\n",
    "embed_size = config[\"embeddings\"][\"embed_size\"]\n",
    "embeddings = init_embeddings(vocab_size, embed_size, name=name)\n",
    "print(\"\\tDone.\")\n",
    "\n",
    "# Build the model and compute losses\n",
    "source_ids = tf.placeholder(tf.int32, [None, None], name=\"source\")\n",
    "target_ids = tf.placeholder(tf.int32, [None, None], name=\"target\")\n",
    "sequence_mask = tf.placeholder(tf.bool, [None, None], name=\"mask\")\n",
    "\n",
    "attn_wrappers = {\n",
    "    \"None\": None,\n",
    "    \"Attention\": AttentionWrapper,\n",
    "}\n",
    "attn_wrapper = attn_wrappers.get(config[\"decoder\"][\"wrapper\"])\n",
    "\n",
    "(enc_num_layers, enc_num_units, enc_cell_type, enc_bidir,\n",
    " dec_num_layers, dec_num_units, dec_cell_type, state_pass,\n",
    " infer_batch_size, infer_type, beam_size, max_iter,\n",
    " attn_num_units, l2_regularize) = get_model_config(config)\n",
    "\n",
    "print(\"Building model architecture ...\")\n",
    "CE, loss, logits, infer_outputs = compute_loss(\n",
    "    source_ids, target_ids, sequence_mask, embeddings,\n",
    "    enc_num_layers, enc_num_units, enc_cell_type, enc_bidir,\n",
    "    dec_num_layers, dec_num_units, dec_cell_type, state_pass,\n",
    "    infer_batch_size, infer_type, beam_size, max_iter,\n",
    "    attn_wrapper, attn_num_units, l2_regularize, name)\n",
    "print(\"\\tDone.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aaronlai/env3/lib/python3.5/site-packages/tensorflow/python/ops/gradients_impl.py:96: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying to restore saved checkpoints from ./log_s2sattn/ ... No checkpoint found.\n"
     ]
    }
   ],
   "source": [
    "# Preparing for training\n",
    "(logdir, restore_from, learning_rate, gpu_fraction, max_checkpoints,\n",
    " train_steps, batch_size, print_every, checkpoint_every, s_filename,\n",
    " t_filename, s_max_leng, t_max_leng, dev_s_filename, dev_t_filename,\n",
    " loss_fig, perp_fig) = get_training_config(config)\n",
    "\n",
    "# Even if we restored the model, we will treat it as new training\n",
    "# if the trained model is written into an arbitrary location.\n",
    "is_overwritten_training = logdir != restore_from\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,\n",
    "                                   epsilon=1e-4)\n",
    "trainable = tf.trainable_variables()\n",
    "optim = optimizer.minimize(loss, var_list=trainable)\n",
    "\n",
    "# Set up session\n",
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=False,\n",
    "                                        gpu_options=gpu_options))\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)\n",
    "\n",
    "# Saver for storing checkpoints of the model.\n",
    "saver = tf.train.Saver(var_list=tf.trainable_variables(),\n",
    "                       max_to_keep=max_checkpoints)\n",
    "\n",
    "try:\n",
    "    saved_global_step = load(saver, sess, restore_from)\n",
    "    if is_overwritten_training or saved_global_step is None:\n",
    "        # The first training step will be saved_global_step + 1,\n",
    "        # therefore we put -1 here for new or overwritten trainings.\n",
    "        saved_global_step = -1\n",
    "\n",
    "except Exception:\n",
    "    print(\"Something went wrong while restoring checkpoint. \"\n",
    "          \"Training is terminated to avoid the overwriting.\")\n",
    "    raise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data ...\n",
      "\tDone.\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "print(\"Loading data ...\")\n",
    "\n",
    "# id_0, id_1, id_2 preserved for SOS, EOS, constant zero padding\n",
    "embed_shift = 3\n",
    "\n",
    "source_data = loadfile(s_filename, is_source=True,\n",
    "                       max_length=s_max_leng) + embed_shift\n",
    "target_data = loadfile(t_filename, is_source=False,\n",
    "                       max_length=t_max_leng) + embed_shift\n",
    "masks = (target_data >= embed_shift)\n",
    "masks = np.append(np.ones([len(masks), 1], dtype=bool), masks, axis=1)\n",
    "masks = masks[:, :-1]\n",
    "\n",
    "n_data = len(source_data)\n",
    "\n",
    "dev_source_data = None\n",
    "if dev_s_filename is not None:\n",
    "    dev_source_data = loadfile(dev_s_filename, is_source=True,\n",
    "                               max_length=s_max_leng) + embed_shift\n",
    "    dev_target_data = loadfile(dev_t_filename, is_source=False,\n",
    "                               max_length=t_max_leng) + embed_shift\n",
    "    dev_masks = (dev_target_data >= embed_shift)\n",
    "    dev_masks = np.append(\n",
    "        np.ones([len(dev_masks), 1], dtype=bool), dev_masks, axis=1)\n",
    "    dev_masks = dev_masks[:, :-1]\n",
    "print(\"\\tDone.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training ...\n",
      "step 0, loss = 6.910712, perp: 1002.608, dev_prep: 1002.761, (0.562 sec/step)\n",
      "Storing checkpoint to ./log_s2sattn/ ... Done.\n",
      "step 20, loss = 6.890607, perp: 979.102, dev_prep: 979.275, (0.166 sec/step)\n",
      "step 40, loss = 6.724417, perp: 820.901, dev_prep: 807.562, (0.160 sec/step)\n",
      "step 60, loss = 6.570716, perp: 702.795, dev_prep: 686.819, (0.156 sec/step)\n",
      "step 80, loss = 6.291841, perp: 529.349, dev_prep: 526.164, (0.160 sec/step)\n",
      "step 100, loss = 6.096406, perp: 437.998, dev_prep: 431.312, (0.155 sec/step)\n",
      "step 120, loss = 5.933227, perp: 379.351, dev_prep: 404.353, (0.153 sec/step)\n",
      "step 140, loss = 5.933444, perp: 369.913, dev_prep: 382.668, (0.155 sec/step)\n",
      "step 160, loss = 5.922973, perp: 367.254, dev_prep: 395.249, (0.153 sec/step)\n",
      "step 180, loss = 5.886409, perp: 355.455, dev_prep: 382.388, (0.163 sec/step)\n",
      "step 200, loss = 5.946142, perp: 368.547, dev_prep: 399.467, (0.165 sec/step)\n",
      "step 220, loss = 5.932016, perp: 371.413, dev_prep: 352.124, (0.167 sec/step)\n",
      "step 240, loss = 5.879278, perp: 352.836, dev_prep: 378.323, (0.168 sec/step)\n",
      "step 260, loss = 5.908895, perp: 360.455, dev_prep: 352.199, (0.164 sec/step)\n",
      "step 280, loss = 5.912663, perp: 363.028, dev_prep: 346.162, (0.154 sec/step)\n",
      "step 300, loss = 5.994283, perp: 395.381, dev_prep: 345.516, (0.160 sec/step)\n",
      "step 320, loss = 5.861436, perp: 352.700, dev_prep: 406.669, (0.159 sec/step)\n",
      "step 340, loss = 5.918251, perp: 370.071, dev_prep: 364.661, (0.167 sec/step)\n",
      "step 360, loss = 5.916575, perp: 365.372, dev_prep: 363.394, (0.164 sec/step)\n",
      "step 380, loss = 5.937312, perp: 365.291, dev_prep: 350.057, (0.146 sec/step)\n",
      "step 400, loss = 5.882322, perp: 353.932, dev_prep: 415.029, (0.164 sec/step)\n",
      "step 420, loss = 5.937707, perp: 374.892, dev_prep: 371.288, (0.156 sec/step)\n",
      "step 440, loss = 5.870573, perp: 350.990, dev_prep: 364.168, (0.157 sec/step)\n",
      "step 460, loss = 5.879701, perp: 352.837, dev_prep: 371.616, (0.158 sec/step)\n",
      "step 480, loss = 5.936172, perp: 373.768, dev_prep: 374.285, (0.157 sec/step)\n",
      "step 500, loss = 5.878153, perp: 351.618, dev_prep: 346.913, (0.159 sec/step)\n",
      "step 520, loss = 5.853421, perp: 344.527, dev_prep: 389.447, (0.150 sec/step)\n",
      "step 540, loss = 5.816163, perp: 332.382, dev_prep: 349.116, (0.158 sec/step)\n",
      "step 560, loss = 5.851508, perp: 332.923, dev_prep: 346.854, (0.164 sec/step)\n",
      "step 580, loss = 5.862463, perp: 349.868, dev_prep: 364.488, (0.165 sec/step)\n",
      "step 600, loss = 5.858058, perp: 342.611, dev_prep: 371.243, (0.155 sec/step)\n",
      "step 620, loss = 5.796992, perp: 325.856, dev_prep: 347.266, (0.157 sec/step)\n",
      "step 640, loss = 5.879349, perp: 346.250, dev_prep: 350.997, (0.155 sec/step)\n",
      "step 660, loss = 5.821054, perp: 316.968, dev_prep: 312.681, (0.155 sec/step)\n",
      "step 680, loss = 5.680297, perp: 285.713, dev_prep: 269.300, (0.155 sec/step)\n",
      "step 700, loss = 5.638410, perp: 266.537, dev_prep: 280.598, (0.158 sec/step)\n",
      "step 720, loss = 5.587600, perp: 246.778, dev_prep: 251.692, (0.156 sec/step)\n",
      "step 740, loss = 5.487586, perp: 236.761, dev_prep: 252.723, (0.159 sec/step)\n",
      "step 760, loss = 5.458277, perp: 229.017, dev_prep: 230.359, (0.162 sec/step)\n",
      "step 780, loss = 5.423844, perp: 229.056, dev_prep: 228.213, (0.166 sec/step)\n",
      "step 800, loss = 5.418821, perp: 217.771, dev_prep: 217.150, (0.164 sec/step)\n",
      "step 820, loss = 5.386518, perp: 213.264, dev_prep: 228.066, (0.170 sec/step)\n",
      "step 840, loss = 5.399931, perp: 213.376, dev_prep: 208.717, (0.166 sec/step)\n",
      "step 860, loss = 5.409641, perp: 213.455, dev_prep: 215.641, (0.165 sec/step)\n",
      "step 880, loss = 5.472206, perp: 218.468, dev_prep: 213.317, (0.161 sec/step)\n",
      "step 900, loss = 5.379126, perp: 207.268, dev_prep: 202.236, (0.157 sec/step)\n",
      "step 920, loss = 5.355635, perp: 204.708, dev_prep: 202.366, (0.151 sec/step)\n",
      "step 940, loss = 5.265712, perp: 185.013, dev_prep: 199.778, (0.158 sec/step)\n",
      "step 960, loss = 5.296997, perp: 190.847, dev_prep: 209.432, (0.146 sec/step)\n",
      "step 980, loss = 5.371487, perp: 195.514, dev_prep: 194.552, (0.168 sec/step)\n",
      "step 1000, loss = 5.261614, perp: 188.843, dev_prep: 194.470, (0.166 sec/step)\n",
      "Storing checkpoint to ./log_s2sattn/ ... Done.\n",
      "step 1020, loss = 5.259672, perp: 186.411, dev_prep: 192.855, (0.153 sec/step)\n",
      "step 1040, loss = 5.295260, perp: 185.271, dev_prep: 187.007, (0.166 sec/step)\n",
      "step 1060, loss = 5.191176, perp: 177.034, dev_prep: 187.327, (0.165 sec/step)\n",
      "step 1080, loss = 5.178194, perp: 169.506, dev_prep: 176.764, (0.156 sec/step)\n",
      "step 1100, loss = 5.172654, perp: 173.100, dev_prep: 175.029, (0.149 sec/step)\n",
      "step 1120, loss = 5.133056, perp: 163.984, dev_prep: 170.751, (0.155 sec/step)\n",
      "step 1140, loss = 5.133612, perp: 164.025, dev_prep: 170.520, (0.156 sec/step)\n",
      "step 1160, loss = 5.111140, perp: 160.326, dev_prep: 160.199, (0.161 sec/step)\n",
      "step 1180, loss = 5.076156, perp: 160.787, dev_prep: 169.074, (0.161 sec/step)\n",
      "step 1200, loss = 5.038260, perp: 156.625, dev_prep: 168.116, (0.164 sec/step)\n",
      "step 1220, loss = 5.028456, perp: 148.169, dev_prep: 154.549, (0.161 sec/step)\n",
      "step 1240, loss = 5.020955, perp: 148.500, dev_prep: 156.516, (0.167 sec/step)\n",
      "step 1260, loss = 4.953352, perp: 136.947, dev_prep: 150.994, (0.165 sec/step)\n",
      "step 1280, loss = 5.234507, perp: 157.904, dev_prep: 157.848, (0.165 sec/step)\n",
      "step 1300, loss = 5.068455, perp: 163.941, dev_prep: 172.780, (0.156 sec/step)\n",
      "step 1320, loss = 4.992486, perp: 142.332, dev_prep: 151.387, (0.151 sec/step)\n",
      "step 1340, loss = 5.030090, perp: 142.237, dev_prep: 151.548, (0.148 sec/step)\n",
      "step 1360, loss = 4.978944, perp: 145.366, dev_prep: 154.649, (0.155 sec/step)\n",
      "step 1380, loss = 4.928328, perp: 135.904, dev_prep: 144.943, (0.167 sec/step)\n",
      "step 1400, loss = 4.934488, perp: 135.848, dev_prep: 147.427, (0.143 sec/step)\n",
      "step 1420, loss = 4.959561, perp: 133.735, dev_prep: 147.470, (0.165 sec/step)\n",
      "step 1440, loss = 4.994024, perp: 143.389, dev_prep: 140.456, (0.167 sec/step)\n",
      "step 1460, loss = 4.992427, perp: 145.726, dev_prep: 139.315, (0.164 sec/step)\n",
      "step 1480, loss = 4.944543, perp: 138.616, dev_prep: 149.598, (0.158 sec/step)\n",
      "step 1500, loss = 4.979425, perp: 143.232, dev_prep: 143.480, (0.161 sec/step)\n",
      "step 1520, loss = 4.880115, perp: 130.752, dev_prep: 140.311, (0.158 sec/step)\n",
      "step 1540, loss = 4.869918, perp: 131.318, dev_prep: 138.555, (0.152 sec/step)\n",
      "step 1560, loss = 4.954102, perp: 129.605, dev_prep: 135.249, (0.165 sec/step)\n",
      "step 1580, loss = 4.954553, perp: 136.960, dev_prep: 140.076, (0.168 sec/step)\n",
      "step 1600, loss = 4.888244, perp: 130.821, dev_prep: 135.345, (0.157 sec/step)\n",
      "step 1620, loss = 4.828527, perp: 121.707, dev_prep: 126.586, (0.155 sec/step)\n",
      "step 1640, loss = 4.813533, perp: 119.986, dev_prep: 130.107, (0.157 sec/step)\n",
      "step 1660, loss = 4.843521, perp: 121.273, dev_prep: 129.833, (0.153 sec/step)\n",
      "step 1680, loss = 4.808959, perp: 118.103, dev_prep: 124.542, (0.145 sec/step)\n",
      "step 1700, loss = 4.821090, perp: 118.185, dev_prep: 124.943, (0.152 sec/step)\n",
      "step 1720, loss = 4.799135, perp: 121.523, dev_prep: 130.043, (0.155 sec/step)\n",
      "step 1740, loss = 4.825497, perp: 122.207, dev_prep: 130.471, (0.158 sec/step)\n",
      "step 1760, loss = 4.781585, perp: 117.391, dev_prep: 124.651, (0.157 sec/step)\n",
      "step 1780, loss = 4.821529, perp: 121.126, dev_prep: 124.377, (0.156 sec/step)\n",
      "step 1800, loss = 4.795683, perp: 116.355, dev_prep: 116.453, (0.154 sec/step)\n",
      "step 1820, loss = 4.758966, perp: 114.283, dev_prep: 124.760, (0.167 sec/step)\n",
      "step 1840, loss = 4.755853, perp: 114.004, dev_prep: 118.358, (0.166 sec/step)\n",
      "step 1860, loss = 4.785057, perp: 113.141, dev_prep: 120.370, (0.164 sec/step)\n",
      "step 1880, loss = 4.780515, perp: 116.097, dev_prep: 122.036, (0.153 sec/step)\n",
      "step 1900, loss = 4.768505, perp: 115.222, dev_prep: 115.966, (0.156 sec/step)\n",
      "step 1920, loss = 4.793819, perp: 117.985, dev_prep: 119.384, (0.157 sec/step)\n",
      "step 1940, loss = 4.781828, perp: 118.280, dev_prep: 119.904, (0.154 sec/step)\n",
      "step 1960, loss = 4.770957, perp: 114.979, dev_prep: 118.207, (0.155 sec/step)\n",
      "step 1980, loss = 4.769724, perp: 116.258, dev_prep: 119.698, (0.154 sec/step)\n",
      "step 2000, loss = 4.742696, perp: 109.247, dev_prep: 113.293, (0.155 sec/step)\n",
      "Storing checkpoint to ./log_s2sattn/ ... Done.\n",
      "step 2020, loss = 4.770551, perp: 114.375, dev_prep: 119.357, (0.154 sec/step)\n",
      "step 2040, loss = 4.777171, perp: 117.373, dev_prep: 125.092, (0.155 sec/step)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 2060, loss = 4.728478, perp: 110.386, dev_prep: 117.075, (0.156 sec/step)\n",
      "step 2080, loss = 4.731610, perp: 111.838, dev_prep: 114.598, (0.155 sec/step)\n",
      "step 2100, loss = 4.691232, perp: 107.956, dev_prep: 119.735, (0.152 sec/step)\n",
      "step 2120, loss = 4.719477, perp: 111.900, dev_prep: 119.932, (0.160 sec/step)\n",
      "step 2140, loss = 4.660933, perp: 103.793, dev_prep: 109.442, (0.150 sec/step)\n",
      "step 2160, loss = 4.711169, perp: 109.563, dev_prep: 115.496, (0.156 sec/step)\n",
      "step 2180, loss = 4.692496, perp: 107.269, dev_prep: 113.517, (0.152 sec/step)\n",
      "step 2200, loss = 4.737661, perp: 108.386, dev_prep: 115.540, (0.153 sec/step)\n",
      "step 2220, loss = 4.714930, perp: 108.321, dev_prep: 113.663, (0.155 sec/step)\n",
      "step 2240, loss = 4.666986, perp: 103.396, dev_prep: 106.385, (0.155 sec/step)\n",
      "step 2260, loss = 4.625841, perp: 100.776, dev_prep: 108.387, (0.161 sec/step)\n",
      "step 2280, loss = 4.618926, perp: 98.408, dev_prep: 107.878, (0.166 sec/step)\n",
      "step 2300, loss = 4.676791, perp: 106.486, dev_prep: 108.284, (0.159 sec/step)\n",
      "step 2320, loss = 4.652904, perp: 102.013, dev_prep: 108.167, (0.155 sec/step)\n",
      "step 2340, loss = 4.673006, perp: 105.399, dev_prep: 105.245, (0.155 sec/step)\n",
      "step 2360, loss = 4.691288, perp: 104.283, dev_prep: 106.112, (0.164 sec/step)\n",
      "step 2380, loss = 4.634536, perp: 100.846, dev_prep: 99.543, (0.167 sec/step)\n",
      "step 2400, loss = 4.579727, perp: 96.543, dev_prep: 104.916, (0.164 sec/step)\n",
      "step 2420, loss = 4.633462, perp: 98.400, dev_prep: 103.358, (0.163 sec/step)\n",
      "step 2440, loss = 4.640957, perp: 99.727, dev_prep: 99.648, (0.165 sec/step)\n",
      "step 2460, loss = 4.594146, perp: 96.615, dev_prep: 102.555, (0.154 sec/step)\n",
      "step 2480, loss = 4.624663, perp: 100.257, dev_prep: 104.528, (0.161 sec/step)\n",
      "step 2500, loss = 4.617896, perp: 97.795, dev_prep: 100.680, (0.156 sec/step)\n",
      "step 2520, loss = 4.555451, perp: 91.862, dev_prep: 96.649, (0.159 sec/step)\n",
      "step 2540, loss = 4.593649, perp: 94.450, dev_prep: 93.450, (0.160 sec/step)\n",
      "step 2560, loss = 4.555036, perp: 92.880, dev_prep: 100.353, (0.147 sec/step)\n",
      "step 2580, loss = 4.517366, perp: 88.413, dev_prep: 96.245, (0.167 sec/step)\n",
      "step 2600, loss = 4.514465, perp: 87.588, dev_prep: 92.475, (0.154 sec/step)\n",
      "step 2620, loss = 4.492311, perp: 87.750, dev_prep: 93.211, (0.159 sec/step)\n",
      "step 2640, loss = 4.485211, perp: 87.252, dev_prep: 89.428, (0.155 sec/step)\n",
      "step 2660, loss = 4.497753, perp: 88.429, dev_prep: 90.244, (0.159 sec/step)\n",
      "step 2680, loss = 4.479930, perp: 87.966, dev_prep: 94.688, (0.151 sec/step)\n",
      "step 2700, loss = 4.508677, perp: 85.123, dev_prep: 89.184, (0.158 sec/step)\n",
      "step 2720, loss = 4.448636, perp: 83.227, dev_prep: 85.734, (0.157 sec/step)\n",
      "step 2740, loss = 4.408039, perp: 79.605, dev_prep: 87.048, (0.167 sec/step)\n",
      "step 2760, loss = 4.510043, perp: 83.865, dev_prep: 87.969, (0.166 sec/step)\n",
      "step 2780, loss = 4.481853, perp: 85.368, dev_prep: 89.461, (0.163 sec/step)\n",
      "step 2800, loss = 4.449749, perp: 84.533, dev_prep: 87.879, (0.156 sec/step)\n",
      "step 2820, loss = 4.451672, perp: 82.929, dev_prep: 84.460, (0.158 sec/step)\n",
      "step 2840, loss = 4.443974, perp: 81.974, dev_prep: 85.919, (0.162 sec/step)\n",
      "step 2860, loss = 4.391865, perp: 80.173, dev_prep: 87.303, (0.164 sec/step)\n",
      "step 2880, loss = 4.395364, perp: 78.832, dev_prep: 80.529, (0.162 sec/step)\n",
      "step 2900, loss = 4.377697, perp: 77.980, dev_prep: 78.972, (0.167 sec/step)\n",
      "step 2920, loss = 4.350756, perp: 75.545, dev_prep: 77.626, (0.164 sec/step)\n",
      "step 2940, loss = 4.407893, perp: 79.095, dev_prep: 75.475, (0.154 sec/step)\n",
      "step 2960, loss = 4.335186, perp: 77.368, dev_prep: 82.721, (0.155 sec/step)\n",
      "step 2980, loss = 4.270627, perp: 70.365, dev_prep: 78.103, (0.157 sec/step)\n",
      "step 3000, loss = 4.319876, perp: 74.627, dev_prep: 77.539, (0.165 sec/step)\n",
      "Storing checkpoint to ./log_s2sattn/ ... Done.\n",
      "step 3020, loss = 4.338101, perp: 71.689, dev_prep: 77.609, (0.150 sec/step)\n",
      "step 3040, loss = 4.348264, perp: 76.890, dev_prep: 82.750, (0.158 sec/step)\n",
      "step 3060, loss = 4.334290, perp: 77.030, dev_prep: 77.659, (0.162 sec/step)\n",
      "step 3080, loss = 4.349840, perp: 72.869, dev_prep: 78.612, (0.167 sec/step)\n",
      "step 3100, loss = 4.372166, perp: 72.369, dev_prep: 76.092, (0.167 sec/step)\n",
      "step 3120, loss = 4.258384, perp: 70.086, dev_prep: 72.667, (0.166 sec/step)\n",
      "step 3140, loss = 4.240941, perp: 68.626, dev_prep: 71.983, (0.163 sec/step)\n",
      "step 3160, loss = 4.222958, perp: 66.316, dev_prep: 72.057, (0.155 sec/step)\n",
      "step 3180, loss = 4.197299, perp: 64.951, dev_prep: 67.643, (0.165 sec/step)\n",
      "step 3200, loss = 4.206738, perp: 66.072, dev_prep: 68.004, (0.157 sec/step)\n",
      "step 3220, loss = 4.196758, perp: 66.310, dev_prep: 69.092, (0.153 sec/step)\n",
      "step 3240, loss = 4.236279, perp: 67.833, dev_prep: 68.972, (0.165 sec/step)\n",
      "step 3260, loss = 4.179068, perp: 64.543, dev_prep: 70.957, (0.155 sec/step)\n",
      "step 3280, loss = 4.163826, perp: 61.445, dev_prep: 66.418, (0.167 sec/step)\n",
      "step 3300, loss = 4.133423, perp: 59.631, dev_prep: 62.923, (0.164 sec/step)\n",
      "step 3320, loss = 4.121133, perp: 61.395, dev_prep: 64.188, (0.157 sec/step)\n",
      "step 3340, loss = 4.205640, perp: 62.964, dev_prep: 65.923, (0.154 sec/step)\n",
      "step 3360, loss = 4.105684, perp: 60.873, dev_prep: 65.065, (0.162 sec/step)\n",
      "step 3380, loss = 4.104165, perp: 58.466, dev_prep: 59.170, (0.162 sec/step)\n",
      "step 3400, loss = 4.031885, perp: 55.567, dev_prep: 60.150, (0.156 sec/step)\n",
      "step 3420, loss = 4.056551, perp: 57.541, dev_prep: 61.095, (0.151 sec/step)\n",
      "step 3440, loss = 4.047289, perp: 59.184, dev_prep: 59.460, (0.158 sec/step)\n",
      "step 3460, loss = 3.989615, perp: 54.315, dev_prep: 59.129, (0.160 sec/step)\n",
      "step 3480, loss = 4.090775, perp: 65.490, dev_prep: 65.590, (0.151 sec/step)\n",
      "step 3500, loss = 4.278530, perp: 72.290, dev_prep: 82.945, (0.165 sec/step)\n",
      "step 3520, loss = 4.070150, perp: 57.716, dev_prep: 61.109, (0.168 sec/step)\n",
      "step 3540, loss = 4.011049, perp: 53.987, dev_prep: 57.333, (0.168 sec/step)\n",
      "step 3560, loss = 3.939045, perp: 49.906, dev_prep: 53.895, (0.164 sec/step)\n",
      "step 3580, loss = 3.940164, perp: 49.659, dev_prep: 51.846, (0.164 sec/step)\n",
      "step 3600, loss = 3.936672, perp: 49.676, dev_prep: 51.163, (0.156 sec/step)\n",
      "step 3620, loss = 3.936729, perp: 49.363, dev_prep: 51.429, (0.155 sec/step)\n",
      "step 3640, loss = 3.887178, perp: 48.344, dev_prep: 52.031, (0.158 sec/step)\n",
      "step 3660, loss = 3.932222, perp: 47.693, dev_prep: 50.633, (0.156 sec/step)\n",
      "step 3680, loss = 3.837934, perp: 44.720, dev_prep: 48.382, (0.152 sec/step)\n",
      "step 3700, loss = 3.852137, perp: 46.467, dev_prep: 48.304, (0.152 sec/step)\n",
      "step 3720, loss = 3.850575, perp: 44.935, dev_prep: 46.279, (0.156 sec/step)\n",
      "step 3740, loss = 3.826153, perp: 47.041, dev_prep: 50.067, (0.151 sec/step)\n",
      "step 3760, loss = 3.793966, perp: 44.595, dev_prep: 49.373, (0.153 sec/step)\n",
      "step 3780, loss = 3.840237, perp: 45.104, dev_prep: 46.834, (0.165 sec/step)\n",
      "step 3800, loss = 3.811066, perp: 45.634, dev_prep: 47.656, (0.165 sec/step)\n",
      "step 3820, loss = 3.806578, perp: 43.739, dev_prep: 45.959, (0.167 sec/step)\n",
      "step 3840, loss = 3.754182, perp: 42.315, dev_prep: 46.284, (0.163 sec/step)\n",
      "step 3860, loss = 3.828917, perp: 42.983, dev_prep: 45.487, (0.148 sec/step)\n",
      "step 3880, loss = 3.753105, perp: 40.661, dev_prep: 42.535, (0.162 sec/step)\n",
      "step 3900, loss = 3.705579, perp: 39.707, dev_prep: 41.901, (0.157 sec/step)\n",
      "step 3920, loss = 3.719224, perp: 39.780, dev_prep: 42.480, (0.155 sec/step)\n",
      "step 3940, loss = 3.693245, perp: 39.406, dev_prep: 40.519, (0.159 sec/step)\n",
      "step 3960, loss = 3.952141, perp: 124.036, dev_prep: 107.061, (0.155 sec/step)\n",
      "step 3980, loss = 3.964271, perp: 50.204, dev_prep: 50.670, (0.164 sec/step)\n",
      "step 4000, loss = 3.876298, perp: 48.392, dev_prep: 54.004, (0.167 sec/step)\n",
      "Storing checkpoint to ./log_s2sattn/ ... Done.\n",
      "step 4020, loss = 3.764068, perp: 41.806, dev_prep: 43.669, (0.163 sec/step)\n",
      "step 4040, loss = 3.725047, perp: 40.675, dev_prep: 41.784, (0.166 sec/step)\n",
      "step 4060, loss = 3.667362, perp: 37.807, dev_prep: 41.849, (0.156 sec/step)\n",
      "step 4080, loss = 3.665541, perp: 37.648, dev_prep: 39.415, (0.155 sec/step)\n",
      "step 4100, loss = 3.608806, perp: 36.132, dev_prep: 39.796, (0.158 sec/step)\n",
      "step 4120, loss = 3.590601, perp: 35.436, dev_prep: 38.366, (0.154 sec/step)\n",
      "step 4140, loss = 3.595501, perp: 35.712, dev_prep: 37.495, (0.161 sec/step)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 4160, loss = 3.592418, perp: 35.064, dev_prep: 35.894, (0.165 sec/step)\n",
      "step 4180, loss = 3.557623, perp: 34.707, dev_prep: 37.302, (0.147 sec/step)\n",
      "step 4200, loss = 3.615348, perp: 35.484, dev_prep: 37.084, (0.163 sec/step)\n",
      "step 4220, loss = 3.601140, perp: 35.598, dev_prep: 36.977, (0.164 sec/step)\n",
      "step 4240, loss = 3.873861, perp: 42.842, dev_prep: 44.658, (0.156 sec/step)\n",
      "step 4260, loss = 3.631117, perp: 36.270, dev_prep: 38.573, (0.157 sec/step)\n",
      "step 4280, loss = 3.662433, perp: 35.630, dev_prep: 36.113, (0.158 sec/step)\n",
      "step 4300, loss = 3.579736, perp: 35.070, dev_prep: 35.912, (0.156 sec/step)\n",
      "step 4320, loss = 3.522473, perp: 32.585, dev_prep: 34.373, (0.162 sec/step)\n",
      "step 4340, loss = 3.485677, perp: 31.947, dev_prep: 33.607, (0.167 sec/step)\n",
      "step 4360, loss = 3.457510, perp: 31.351, dev_prep: 33.095, (0.165 sec/step)\n",
      "step 4380, loss = 3.450241, perp: 30.744, dev_prep: 32.595, (0.166 sec/step)\n",
      "step 4400, loss = 3.458753, perp: 31.085, dev_prep: 31.969, (0.162 sec/step)\n",
      "step 4420, loss = 3.423163, perp: 29.893, dev_prep: 31.429, (0.150 sec/step)\n",
      "step 4440, loss = 3.456233, perp: 30.096, dev_prep: 31.159, (0.155 sec/step)\n",
      "step 4460, loss = 3.397536, perp: 28.877, dev_prep: 32.461, (0.161 sec/step)\n",
      "step 4480, loss = 3.418253, perp: 30.079, dev_prep: 31.849, (0.153 sec/step)\n",
      "step 4500, loss = 3.451395, perp: 30.432, dev_prep: 32.327, (0.158 sec/step)\n",
      "step 4520, loss = 3.420583, perp: 30.274, dev_prep: 31.348, (0.157 sec/step)\n",
      "step 4540, loss = 3.429104, perp: 31.020, dev_prep: 32.393, (0.154 sec/step)\n",
      "step 4560, loss = 3.486289, perp: 31.762, dev_prep: 33.201, (0.157 sec/step)\n",
      "step 4580, loss = 3.432981, perp: 29.510, dev_prep: 30.826, (0.156 sec/step)\n",
      "step 4600, loss = 3.347626, perp: 27.577, dev_prep: 30.213, (0.168 sec/step)\n",
      "step 4620, loss = 3.363311, perp: 28.091, dev_prep: 29.401, (0.155 sec/step)\n",
      "step 4640, loss = 3.319120, perp: 27.138, dev_prep: 28.468, (0.153 sec/step)\n",
      "step 4660, loss = 3.347230, perp: 27.293, dev_prep: 28.840, (0.163 sec/step)\n",
      "step 4680, loss = 3.350195, perp: 27.043, dev_prep: 29.322, (0.164 sec/step)\n",
      "step 4700, loss = 4.273815, perp: 84.664, dev_prep: 90.158, (0.165 sec/step)\n",
      "step 4720, loss = 3.663738, perp: 35.428, dev_prep: 37.757, (0.165 sec/step)\n",
      "step 4740, loss = 3.397637, perp: 28.425, dev_prep: 29.851, (0.167 sec/step)\n",
      "step 4760, loss = 3.346877, perp: 27.375, dev_prep: 28.632, (0.162 sec/step)\n",
      "step 4780, loss = 3.302194, perp: 27.278, dev_prep: 31.091, (0.155 sec/step)\n",
      "step 4800, loss = 3.283494, perp: 26.076, dev_prep: 28.356, (0.160 sec/step)\n",
      "step 4820, loss = 3.243557, perp: 25.555, dev_prep: 28.201, (0.153 sec/step)\n",
      "step 4840, loss = 3.287204, perp: 25.993, dev_prep: 28.135, (0.154 sec/step)\n",
      "step 4860, loss = 3.231910, perp: 24.710, dev_prep: 26.747, (0.157 sec/step)\n",
      "step 4880, loss = 3.237643, perp: 25.053, dev_prep: 27.156, (0.154 sec/step)\n",
      "step 4900, loss = 3.255785, perp: 25.731, dev_prep: 27.080, (0.160 sec/step)\n",
      "step 4920, loss = 3.212458, perp: 24.129, dev_prep: 26.069, (0.150 sec/step)\n",
      "step 4940, loss = 3.207682, perp: 24.214, dev_prep: 25.613, (0.166 sec/step)\n",
      "step 4960, loss = 3.205015, perp: 23.565, dev_prep: 25.872, (0.163 sec/step)\n",
      "step 4980, loss = 3.204172, perp: 24.390, dev_prep: 26.297, (0.164 sec/step)\n",
      "Storing checkpoint to ./log_s2sattn/ ... Done.\n"
     ]
    }
   ],
   "source": [
    "last_saved_step = saved_global_step\n",
    "num_steps = saved_global_step + train_steps\n",
    "losses = []\n",
    "steps = []\n",
    "perps = []\n",
    "dev_perps = []\n",
    "\n",
    "print(\"Start training ...\")\n",
    "try:\n",
    "    for step in range(saved_global_step + 1, num_steps):\n",
    "        start_time = time.time()\n",
    "        rand_indexes = np.random.choice(n_data, batch_size)\n",
    "        source_batch = source_data[rand_indexes]\n",
    "        target_batch = target_data[rand_indexes]\n",
    "        mask_batch = masks[rand_indexes]\n",
    "\n",
    "        feed_dict = {\n",
    "            source_ids: source_batch,\n",
    "            target_ids: target_batch,\n",
    "            sequence_mask: mask_batch,\n",
    "        }\n",
    "\n",
    "        loss_value, _ = sess.run([loss, optim], feed_dict=feed_dict)\n",
    "        losses.append(loss_value)\n",
    "\n",
    "        duration = time.time() - start_time\n",
    "\n",
    "        if step % print_every == 0:\n",
    "            # train perplexity\n",
    "            t_perp = compute_perplexity(sess, CE, mask_batch, feed_dict)\n",
    "            perps.append(t_perp)\n",
    "\n",
    "            # dev perplexity\n",
    "            dev_str = \"\"\n",
    "            if dev_source_data is not None:\n",
    "                dev_inds = np.random.choice(\n",
    "                    len(dev_source_data), batch_size)\n",
    "                dev_feed_dict = {\n",
    "                    source_ids: dev_source_data[dev_inds],\n",
    "                    target_ids: dev_target_data[dev_inds],\n",
    "                    sequence_mask: dev_masks[dev_inds],\n",
    "                }\n",
    "                dev_perp = compute_perplexity(\n",
    "                    sess, CE, dev_masks[dev_inds], dev_feed_dict)\n",
    "                dev_perps.append(dev_perp)\n",
    "                dev_str = \"dev_prep: {:.3f}, \".format(dev_perp)\n",
    "\n",
    "            steps.append(step)\n",
    "            info = 'step {:d}, loss = {:.6f}, '\n",
    "            info += 'perp: {:.3f}, {}({:.3f} sec/step)'\n",
    "            print(info.format(step, loss_value, t_perp, dev_str, duration))\n",
    "\n",
    "        if step % checkpoint_every == 0:\n",
    "            save(saver, sess, logdir, step)\n",
    "            last_saved_step = step\n",
    "\n",
    "except KeyboardInterrupt:\n",
    "    # Introduce a line break after ^C so save message is on its own line.\n",
    "    print()\n",
    "\n",
    "finally:\n",
    "    if step > last_saved_step:\n",
    "        save(saver, sess, logdir, step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEWCAYAAACdaNcBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VdXV+PHvykQIhDCFeQgIojIKUQEVRRAZrHbQvljfOrxaarVOtb8W1OKstLZ1aK0WcdYqVatSQWaUQRnCPM9hFBLGEIZAkvX7455cbm5ukpvkzlmf58nDufvsc84+Ma7s7LPP2qKqGGOMiS1x4W6AMcaYwLPgbowxMciCuzHGxCAL7sYYE4MsuBtjTAyy4G6MMTHIgrsxHkQkWURURNqUs3+hiPxvqNtlTFVZcDcRT0TyPb6KReSkx+ebKzl2qIhsCVVbjYkUCeFugDGVUdX6Jdsikg3cqaozw9ciYyKf9dxN1BORuiLyioh8LyK7ReR5EUkUkSbAZ0BHj55+ExG5VEQWicgREdkrIi+ISJU7OiISLyJPiMhOEdkvIm+KSKqzr56IfCQih5zrLBKRRs6+X4hItogcE5FtInJjYL8jxlhwN7HhCaAH0B3oA1wJ/E5VDwI/Arapan3n6yBwBvg10AS4HPgBcGc1rvtL4KfOOToDzYC/OvvuxPWXcWugqXO9006Afx4YpKqpwGXAmmpc25gKWXA3seBm4DFVPaCq+4GngZ+XV1lVF6vqElUtUtWtwATgimpe93lV3aGqecAjwM0iIrh+gaQD56hqoXO94x7HdhORZFXdq6rrq3FtYypkwd1ENSeQtgB2eBTvwNVjLu+YC0TkK2coJQ8Yi6t3XVWtfFy3LtAYeAP4BvjEGSp6VkTiVfUwrl8K9wH7RGSSiHSqxrWNqZAFdxPV1JXWdB/Q3qO4HbCnpIqPw14HluHqVTcAngSkGpff6+O6J4FDqlqgqmNV9TxgAHAjMNJp82RVHYTrl8NO4NVqXNuYCllwN7HgQ+Ax52FpM1zDI+87+/YDzUSkvkf9VOCoquaLSFfgFzW47m9FpJ3zIPVp4F+qqiIy2PkLIQ7IAwqBYhFpLSIjRCQFKADygeJqXt+YcllwN7FgLLAOWAusABYAf3L2rQQmATucWSuNgQeBO0UkH3gFmFjN674K/Af4FtgKHAJ+4+xrDXwBHMP1wHSKc514YDSuvzYOAhfhethqTECJLdZhjDGxx3ruxhgTgyy4G2NMDLLgbowxMciCuzHGxKCwJQ5r2rSpZmRkhOvyxhgTlZYuXXpAVdMrqxe24J6RkUFWVla4Lm+MMVFJRHZUXsuPYRkR6SIiKzy+8kTkAa86IiIvi8gWEVklIr2r23BjjDE1V2nPXVU3Ar3AleIU12vdn3lVG4YrK15n4BJcL3dcEtCWGmOM8VtVH6gOAraqqvefBdcD76rLQqChiLQMSAuNMcZUWVWD+0hc+TS8tQZ2eXzeTQVZ+YwxxgSX38FdRJKA64CPq3sxERklIlkikpWbm1vd0xhjjKlEVXruw4BlzmII3vYAbT0+t+FsylU3VR2vqpmqmpmeXulMHmOMMdVUleB+E76HZMCVde8WZ9ZMX1zpVL+vceuMMcZUi1/BXUTqAVfjSm9aUnaXiNzlfJwCbAO24FoI4e4At9Ntz5GT/HHqBnYdOhGsSxhjTNTz6yUmZ+3HJl5lr3lsK3BPYJvm29Idh/nnN1v5OGsXU+6/nGapyaG4rDHGRJWoyy1zXc9WTPr1ZRzIP82kFXvD3RxjjIlIURfcAbq2akBKUjx7j5wKd1OMMSYiRWVwFxFapCWzP8+CuzHG+BKVwR2gRYNk9llwN8YYn6I2uDetX4eD+QXhboYxxkSkqA3uDVMSOXLyTLibYYwxESl6g3vdRI6cOMPsDb5emDXGmNotaoN73qlCAP7vbVvwwxhjvEVtcL+0U9NwN8EYYyJW1Ab3gV0s8ZgxxpQnaoN7QvzZpruyHxhjjCkRtcHd09q9eeFugjHGRJSYCO4Hj58OdxOMMSaiRHVw/+tPewJw13tLw9wSY4yJLFEd3EvS/Z48UxTmlhhjTGSJ6uDe/5wmlVcyxphaKKqDe1ychLsJxhgTkaI6uBtjjPHN3zVUG4rIJyKyQUTWi0g/r/1XishREVnhfI0NTnPLGtglnWapdUJ1OWOMiQp+raEKvARMVdUbRCQJSPFRZ56qXhu4pvmnfZN6ZGUfDvVljTEmolUa3EUkDRgA3AagqqeBiJlY3igliWMFhZwpKiYx3kaZjDEG/BuW6QDkAm+JyHIRmSAi9XzU6yciK0XkKxHp6utEIjJKRLJEJCs3N7cm7XZrVC8RgCMnLLe7McaU8Ce4JwC9gVdV9ULgODDaq84yoL2q9gT+Bnzu60SqOl5VM1U1Mz09MIm/GqYkAXDkRMT8MWGMMWHnT3DfDexW1UXO509wBXs3Vc1T1XxnewqQKCIhycnbsK7Tc7dVmYwxxq3S4K6q+4BdItLFKRoErPOsIyItRESc7Yud8x4McFt9apjiCu6b9+eH4nLGGBMV/H0CeS/wgYisAnoBz4rIXSJyl7P/BmCNiKwEXgZGaojy8DZyhmXeW7gjFJczxpio4NdUSFVdAWR6Fb/msf/vwN8D2C6/NanvCu7rv7e0v8YYUyLq5w4m2fRHY4wpI+ojY8mKTD3apIX82h8s2sHWXBvrN8ZEnqgP7gCXd26K8zw3pB75bA0jXp4X8usaY0xlYiK4N0pJ4nCYVmM6daY4LNc1xpiKxERwb9kwmX1HT1FcbAtlG2MMxEhwb56azOmiYo6G8EWmEM30NMaYaomJ4L73yEkAXpi5KWTXtNhujIlkMRHc9x8rAGDupsAkI/OHxXZjTCSLieDesakrSWXLtLohu6YNyxhjIllMBPfb+mcAMPiC5iG7poV2Y0wki4ngXjcpHoCnvlxXSc3AsY67MSaSxURwD0cKArW+uzEmgsVEcI+LC/3bqdZzN8ZEspgI7p4OhelNVWOMiSQxE9xLHqr2fmpGSK5nPXdjTCSLmeA+b3Po5riDjbkbYyJbzAT35MT4kF7Peu7GmEjmV3AXkYYi8omIbBCR9SLSz2u/iMjLIrJFRFaJSO/yzhUsv7n6XPf20h2HuOPtJRQVK6rKyl1HAn4979iuqhzILwj4dYwxpjr87bm/BExV1fOAnsB6r/3DgM7O1yjg1YC10E9tG6e4t3/y6nfM2pDDY5PW8NiktVz/ygKmr91Xo/Nv3HeMS56d6Q7g3m+ofrh4F5lPz2TDPlvuzxgTfpUGdxFJAwYAbwCo6mlV9e4KXw+8qy4LgYYi0jLgra1Aoo+57u8v3Mm737kWzh713lKW7TzM0RNneH/hDpbuOATAqTNFfp1/wrxt7M8rYNb6/UDpnvvRk2eYv8U15r8153gN7sIYYwLDnwWyOwC5wFsi0hNYCtyvqp5RrDWwy+Pzbqfse88TicgoXD172rVrV4Nml9XUWSi7Is9NWc/B46fZlls6AH9572V0a312mb473l5C5+apnNcilet7tWJrbj4fL90NwKb9rmX1PDvux06d4cRp1y+Jbbn5FBQWUSchtM8AjDHGkz/BPQHoDdyrqotE5CVgNPCHql5MVccD4wEyMzMD+kgyNTmx0jpLsg/7LL/2b/Pp2SaNlbuP0rtdQ5btPMKsDTkAbNh3jNe+2equ+8b87azafYQJt1zkLss9VsDXG10997/M2MTmnHxevunCmtyOMcbUiD9j7ruB3aq6yPn8Ca5g72kP0NbjcxunLGqs3H0UgGU7S484eQb2EkuyD3PLW4vdn3/0j29L7Z+0cm8QWmiMMf6rNLir6j5gl4h0cYoGAd4ZuiYBtzizZvoCR1X1e2JYMGbgGGNMoPg7W+Ze4AMRWQX0Ap4VkbtE5C5n/xRgG7AFeB24O+At9cOSRwaH47LGGBNx/BlzR1VXAJlexa957FfgngC2q1rSU+uEuwnGGBMRYuYN1RKT77ss3E0wxpiwi7ng3rVVGtnjRoS7GcYYE1YxF9xLrHxsCNMeGBC2609dU7M3Yo0xpiZiNrin1U2kS4tUsseNYPZDV4T8+ne9vzTk1zTGmBJ+PVCNdh3T67Pq8SHsPHiCz5bv4Y3523lw8LmkJMXziwEdKSwqpvOjX6Hqygu/ePsh/nxjT0Rg2Evzqn3d04XFJCXE7O9PY0wEE+8EWKGSmZmpWVlZYbm2L4VFxby1IJtb+rcvkzogY/RkAIZ3b8G0tfspKj77PWvRIJl9ead8nnP140P8enPWGGP8JSJLVdV79mIZ1q10JMTH8YsBHX3mhPnTT3oAcG2PVmx9dnipfT+8sHW55/xmUy4nT/uXmMwYYwLJeu5+ysk7RbMGyQDMXLeffy3eyaMjzqdd4xS6Pz6dk052yTm/vZKBf/7afVzT+klkPXp1OJpsjIlB1nMPsJLADjD4gua8edtFdEyvT0J8HK/c7EoSdt9VnejQtF6p4w7k24LdxpjQqxUPVINtYJdmjP95Hwae1yzcTTHGGMCCe0CICEO6tgh3M4wxxs2GZYLginPTw90EY0wtZ8E9CHq3axTuJhhjajkL7kFQN6n0t9WmQxpjQs2CexAIUurzE/9dG6aWGGNqKwvuQaCUfnfgoyW7KC4Oz/sExpjayYJ7EFzXs+xbqze9vjAMLTHG1FYW3IOgRVoyQ72mRi7afihMrTHG1EZ+BXcRyRaR1SKyQkTK5AwQkStF5Kizf4WIjA18U6PL8zf2KFP2xvztZIyezJyNOWFokTGmNqlKz32gqvaqIKfBPGd/L1V9MhCNi2apyYksfbT0gt1PfbkOgNvfWhKOJhljahEblgmiJvXr0DItufKKxhgTYP4GdwWmi8hSERlVTp1+IrJSRL4Ska6+KojIKBHJEpGs3NzcajU42nz9/670Wf7tlgOs2XPU/fnd77LJGD2ZU2dsTrwxpub8De6XqWpvYBhwj4h4L066DGivqj2BvwGf+zqJqo5X1UxVzUxPrx2v6PvKDw/wswmLuPZv83nn22wAxn7hmgt/IL8gVE0zxsQwv4K7qu5x/s0BPgMu9tqfp6r5zvYUIFFEmga4rVHLe4EPT49NWsu6vXnuzyJSbl1jjPFXpcFdROqJSGrJNjAEWONVp4U4UUlELnbOezDwzY1O8XEVB+zhL59dp9VCuzEmEPzpuTcH5ovISmAxMFlVp4rIXSJyl1PnBmCNU+dlYKSGa4mnCOU977087y/cgaoyfe0+Xp+7jV//a1mQW2aMiUW2zF6IbMk5xsjxi/waU29aP6nUCk7rnryGlCRLvW+MsWX2Ik6nZqlkec17L4/30nzXvDg3GE0yxsQwC+4h9u9f9iOjSUqVjtl16CSvfr2VXYdOBKlVxphYY8MyYVJUrNz30XImr/q+SsdljxsRpBYZY6KBDctEuPg44ZWf9SYlyfc8eGOMqQkL7mG27smhVap/6bjZfLZ8d5BaY4yJFRbcI8Bbt1/kd909R07y4MSVvPdddtDaY4yJfhbcI8DALs349Ff9ad8khX/+vI9fx/zhi6ov3bc1N7/KxxhjopM9UI1ABYVFdHl0ql91L+3UhA/u7FtpvWenrGf83G38NLMNf7qhZ02baIwJE3ugGsXqJMTz4v/0AuDb0VdVWHfBloMUFBaxbOfhCuuNn7sNgH9n2Xi9MbWBBfcI9cMLW5M9bgStGtbl37/sV2Hd30xcyY//8S0fLt4ZotYZYyKdBfcocHGHxhXun7zaNVf+uSnrUVVenrWZnLxToWiaMSZCWXCPEs/+qHuldfJOFbJq91H+OmMT93+0otx6z3213hYFMSbGWTaqKHHTxW3p0iKVtLqJDP7rN+XWu/6VBQB8t638jMv//GYbaXUTufvKTgFvpzEmMlhwjxIiQp/2jYCzKQhyjp3i4mdmlXtM7rEC0lPr+NxXcKaYv83azPW9WtOuirlujDGRz4ZloljTer4Dd4mhL87lznd8TzedtnYff5mxidvfXhyMphljwsyCexSLq2SFp4PHTzNz/X6OFxSW2bdh3zEATpy2sXdjYpEF9yiXPW4EV53XrMI6XR+bVu6+74/arBoTekXFii3WFlx+BXcRyRaR1SKyQkTK/J0vLi+LyBYRWSUivQPfVFOeN2+7qEapgPNOnfGrXn5BIXd/sJTcY5WvJmVMRc55eAr3frg83M2IaVXpuQ9U1V7lvPY6DOjsfI0CXg1E40zVvDSyV7WO6/H4dL/qfZK1iymr9/G32ZurdR1jPH1ZxbUMTNUEarbM9cC7zqLYC0WkoYi0VFX7rxdC1/dqzbvf7WDpjopTEfiSMXoy4Fq/dUT3lrzz3Q5Wjh1C/ulCWjesC7hm7BhjooO/PXcFpovIUhEZ5WN/a2CXx+fdTlkpIjJKRLJEJCs3N7fqrTWVurV/Ro2OP5B/mne+2wFAzyenc+m42ZwpKgagJLa/6+w3xkQuf4P7ZaraG9fwyz0iMqA6F1PV8aqaqaqZ6enp1TmFqcQVndM5J70evxvahU9/1Z8ebdJK7X/vjou5rFPTKp2z8yNfccfbSyg4UxzIphpjgsiv4K6qe5x/c4DPgIu9quwB2np8buOUmRBLS0lk1kNXcveVnejTvhF/vrF0et/LOjXl/TsvqfJ5Z23I4Zkp692flztZKAuLiikuPjvrQVU5adMrjQm7SoO7iNQTkdSSbWAIsMar2iTgFmfWTF/gqI23R4Zzm6cycdTZfO8l4+aLHh7EhqeGkpxYvdmwN09YBECnR77ioY9Xusv/OmMT54+dSr6PufXGmNDx5//s5sB8EVkJLAYmq+pUEblLRO5y6kwBtgFbgNeBu4PSWlMtl3RswoanhjL/9wPdZc0bJJOcGM9vh3Sp1jlPnC5i+4HjAHy2/OwfaZ8udeWLP3LidA1aXHV7j5zkmJ9TOo2pDSqdLaOq24AyS/eo6mse2wrcE9immUBKToynTaOyOWTuvLwjV3ZJ55z0+nQYM6VK5xz456/d26qKiLj/MhjywlxWP34N8ZW8RRso/cfNpmPTesz+7ZUhuZ4xkc7eUDV0apaKiJA9bgTzfjeQtU9cw+WdXQ9dM51kZZXpMGYKx06dcc+oOXG6iBOnXUMzG/blkTF6MnM3BXeG1DbnLwljjAV346Vt4xTq1UngvTsuYd7vBvLRqL78ckBHv47t/vh0dh8+Wepz7rECFm8/BMD0dfsoKCxixrr9QWm7MeYsC+6mXG0bp5AQH8eDV5/rLnv15t7cP6iz3+e46JmZ7h77+wt3cu+/lvOLd7PIGD2Z/c5qUUuyD5WacbNubx79n5vF4eOn2X7gOIu2HbQ8JMZUkeVzN5VKTowvlbtmWPeWvDTL/xQEM9fnuLene/Ta//D5Gi7p2ISnvlxH52b1mXL/5RQWKcNfngfA+HnbePXrrQA8cV3XGr+gZSJDzjFLVhcK1nM31VJZJkp/TF+3n6e+XAfA5px8Oj/yFb98f6l7f0lgB/h46a4yx5voZC/DhYYFd1Mtvds1BGDyfZcF9LzlPXRdsyePBz5azpo9R1mw5QCfLN3NV6u/d4/nl9h58ERA22NMtLJhGVMtd1/ZiasvaEGXFqkA3NinDfM2H2BfXvD+5P58xV4+X7G3TPmSRwa7twc8P6fUENKd7yxh1oYctj9X/ZTIxkQjC+6mWuLixB3YvXPJqyovz9rCCzM3haQtFz0zs0zZvqOnOHbqTKnxfmNqEwvuJuBEhPsHd2ZI1+Zs2n+Myzo1pc/TZQNwsDz82Wr+tWhnyK5nTCSy4G6C5vyWDTi/ZQPA1bvfdegEB/IL6Ni0PpnPzOBMUXCmN/oK7He9t5SUpHgev74rDZITg3JdYyKJBXcTMm0bp9C2sSsFwvzfX8XUNfs4U1RMq4Z1ufuDZe56N/Zpw8dOjppAmbp2HwD/Wb6Hvh0b89ef9qJlWrItQGJilgV3ExbNGySXmrc+/cEBLN1xmHOb16dP+8Ys3H6QXYdcb7t2bdWAtXvzABjatYU7UFfXwm2H6D9uNgAjL2rLuJ/0AOCZyet4fd72Gq1HG40KCl0pmuskxIe5JSaQLLibiHBu81TObZ7q/jzjwSs47w9T+WGvVrw48kJ2HTpBcmI86al1AFfWyV5PznDXb5SSyL9/2Y+rX5hbpet+tGQXe4+eIjkhrtQLVuBaevCc9HrMeujK6t9YFOjz1ExOFxaz6Zlh4W6KCSAL7iYieb8VWzKcU6JhShIrHxtCvaR4th84TmePXwxV5T23PmP0ZC5wnhVszQ19MrKH/r0SEcostBIsoc69b5kkQsNeYjJRK61uIgnxcaUC+9QHLi9Tb1i3FlU+97rv89zb5eWmf3DiCp523rANpE+X7eaTAD9z8Idnfh8T/aznbmLKeS0auHv8Ow4eZ0n2YW7o04YJ87bx9OT1lRztW68nZ7D44UE0a5DsLjuYX+BepOTRay+oecMjwJ4jJ8v8hWSil/XcTcxq36QeN/RpA7gWJfninkurfa7/98kq8jxWegrlvP1Yo9hfCKFgPXdTa/Rs25CFYwYRFwf3f7iC77Yd9PvYbzbl0uPx6XRuVp/LnIVMSvz4HwtoWr8O42/J5NutB9iae5xh3VqgivsBcDSwsfDY4ndwF5F4IAvYo6rXeu27DXgeKFlM8++qOiFQjTQmUFqkuYZWPhzVl5xjp2hYN4np6/ZxXosGqCoT5m1nYlb5GSg35+SzOSe/VNmynUfc2z973bVw+B8+d60hnz1uBD/+xwIU+Ozuyv9yCGfeeutRx5aq9NzvB9YDDcrZP1FVf13zJhkTGs1SXYH+2h6t3GXjftKd63u1YumOwwzp2oJrXvR/auUdby8pU3amqLhU8K9MOHvPC7YcpH2TeuFrgAkov8bcRaQNMAKw3riJaSJC/05NuXdQZ7q0SGXrs8P9PnbWhrJJym549dsqXb84jNF9+c7DYbu2CTx/H6i+CPwOqCjL/k9EZJWIfCIibX1VEJFRIpIlIlm5ucFdLNmYQIiPExY/PKjax6/cfdS9PeSFbyqt7xnaQz01MS5EqRhsbD80Kg3uInItkKOqSyuo9l8gQ1V7ADOAd3xVUtXxqpqpqpnp6enVarAxodasQTKrHx9S4/Ns2p9faR3PnntRiKOgpdmJLf703C8FrhORbOAj4CoRed+zgqoeVNUC5+MEoE9AW2lMmKUmJ7LkkcH0bJNWo/Ns3n+swh753E0H3NsnCopqdK2qio+z6B5LKg3uqjpGVduoagYwEpitqv/rWUdEWnp8vA7Xg1djYkp6ah2++PVlTHtgAACP/+ACvrq/7BuxFbn6hbmMfH1huftX7z778PXhz1dXr6HVdCC/oPJKJmpU+yUmEXlSRK5zPt4nImtFZCVwH3BbIBpnTCTq0iKV7HEjuO3SDpzf0vVG7L9/2c/v4xdvP8SpM0U88d+1ZIyezPYDxylyevOeKYjnbcpFVdmwL6+8UwXUtLX7K68UADbkHhpVeolJVb8Gvna2x3qUjwHGBLJhxkSTizs0Zv7vBzJrfQ79z2lCbn6Be867L+f9Yap7e+Cfv+bnfdvz1A+78eb87e7yvFOF/GfZHh76eCVv3XYRA89rFtR7MBXbn3eK5h4pKCKdpR8wJkDaNErh1v4ZdG6eSv9zmrLysSHMeugK/nFzb66qJDC/t3AHAMe8MjT+ZfpGAG73MYfehM7EJTu55NlZ/P6TVeFuit8suBsTJGl1EzknvT7Du7ekVcPKe3wZoyeXKdt79JR7+0xRRTORo0c438KtrqlrXAvEVPT2cqSx4G5MCDw8/Hz+UsP87DsOnghQa0xVReNyjBbcjQmBlKQEftKnDSMv8vl+n1/m+HgD1tOeIyfdD2bB1UM+fNx3LnpTNdE4S9SCuzEh9NyPuzPlvstZOGYQr/1vbyb92ncyscn3XVam7Jkp6/l6o+8Av+vQCS4dN5uXZ212l732zTYufGoGi7cfCkzjazHruRtjKiQiXNCqAS3SkhnarSU92jQssyD35/dcStdWaT4X6r7tLd8PVvcccS0m7pnG+I9TNwBw+1uLy22Prb7kH+u5G2Oq5f8u7cCjI84ne9wIerVt6C5fOGYQl3vlj/d88LrnyElembOFb7e43mxN8BGFzhSVH8DDmagsmoQq704g2WIdxkSAsT/wvVRfi7RkJtyaSZdHp5Yqf3P+dnq1a8iP/1E66+S3Ww+iqqWGESoK4OHouNuvk9CwnrsxEa5OQjzbnyudevjJL9eVCewl1uwp/UZrYQURPFamV5qyLLgbEwWq8kDvB3+f73fQfsnjAWw4RMuc9yhpZikW3I2JEmueuIbzW5a3EFppnR/5yq96q3aXXiWqMAQ9ec9Aac9zg8eCuzFRon6dBL66/3JWjL2ap37YrUrH+jsdMtTTJouiJLpH4/qyFtyNiTINU5L4ed/23HRxO26+pB2/uLxDpcf89J/f+Sz3Hm44GOKXnqJlab9QZcwMJJstY0yUeu7H3d3bt/TLoGVaMne9v4yZ630HoqlrvmdoN9fSC3M35bJm79Eyde79cDk/6NmqTHlgnf2NMndzLpd0bBLk69VO1nM3Jga0bZxCQnwcE27NZPZDV/isc9f7y9wPMG95czF/mrox7IMN8zYfqLySqRYL7sbEmI7p9WmUkgjAoyPOL7Wvw5gpdHp4ytkCH9F9zZ6yPfpA8hwKWrU7uNeqzSy4GxODljwymFkPXcGdl3css89z3nvT1CSAUm/FXvu3+UFtm+fvk4Fd0oN6rdrM7+AuIvEislxEvvSxr46ITBSRLSKySEQyAtlIY0zVJMTHcU56fYAyL0B5mrLayVP+y741ut7xgkKOey004o/2TepV63pHTpxm1LtZHDlhWS/LU5We+/2Uv/D1HcBhVe0EvAD8saYNM8YEhoj4TELmKSm+dCiYtX4/6/b6v3Zr18em0fWxaX7V9RyWad2wrt/X8PTm/O1MX7eft7/NrtbxtYFfwV1E2gAjgAnlVLkeeMfZ/gQYJNGYI9OYGNaleWq5+0SEBwZ3dn++450shr88r1SdomJloUfWyerynDP+zJTy+ov+ESzMlMffnvuLwO+A8l5faw3sAlDVQuAoUGZ+k4iMEpEsEcnKzc1AZQPwAAARCklEQVStRnONMdX1/p2X8I+be7Pq8SE87iNR2S98jM/vzzu7zN/fZ29h5PiFAQnwJvgqDe4ici2Qo6pLa3oxVR2vqpmqmpmebg9SjAml9NQ6DO/ekgbJidx2aQfeuDWTe6/qxKanhwFQr04CbRuXHia55NlZ7u2tufkAfH/0ZI3aEYg8LeGewhkN/Om5XwpcJyLZwEfAVSLyvledPUBbABFJANIA+/VuTAQbdH5zHhrShaSEs2Fg6v0DytSbsyGHjNGTmbRyLwDFXn+/VzX5VyCTcNngb/kqDe6qOkZV26hqBjASmK2q/+tVbRJwq7N9g1PHfrkaE2Xq1Sn70vrtb5de/anI63/tihYD8SUa87REo2rPcxeRJ0XkOufjG0ATEdkC/AYYHYjGGWNC7z93969w/76jp0p9LvToyq/Zc9S9KlR5vLt909fuq1oDfZzDlFWl4K6qX6vqtc72WFWd5GyfUtUbVbWTql6sqtuC0VhjTPD1bteI//667ALdJeZuykVV3euvTvMIztf+bT4/m7CoStcb9V6NH+cZH+wNVWNMGV1bNaBOgu/wkLXjMB3GTKHjw1PYd/QUD05c6bPeil1HeG/hjjLlvpb9q+pD2pKhnSMnzlTpuNrEgrsxpoy4OGHj08PIHjeCGQ8O4LwWvufI931uls/y/IJCfvjKAv7w+Zoy+0piu+c5t+TkV6l9y3a4Fhl5c8H2Kh1Xm1hwN8ZUqHPzVKY+MKDCNAbeVu06Uu6+kn57u8Yp7rL3vivbw69IRYt+GxcL7sYYv4gIK8cO8auu57j7p0t3l9pXEpjrJMa7yzo6eXD8lRBvcyArY8HdGOO3tJREnr+hR5WOeejj0mPyJZ3uHq3Tqt2OhDgLXZWxlZiMMVVyY2ZbUpMTmLMhl4lZu9zlMx4cwOacfO7+YFmFx5e8AnNey7Nj7rsPn6hSGxKt514p+/VnjKmyod1aMvYHF9AxvR6f3d2fzc8Mo3PzVIZ3b+mz/v+9vcQd1EtGywVhweirAPhy1fecLiwvdVVZ2w4cr1H7awML7saYaqlXJ4HZD13Jhe0akeiRMvhPPoZtZm/I4ZtNrmSBJcMyIqVT/j4/bYPf196WG77gXlQcHQ9zbVjGGBNQP81sy5ETpzl5upgXZm5yl9/2liuNQUlq4SSvefSvz9vOQ0O6kOzxoNUX7zdgl+44RN7JQgae1ywQza/UmaJi4uMqbmMksJ67MSbgRg04h/sHd2bDU0PL7Htx5mYA6voI4o99sbbScx/yWn3pJ69+Vyb/TTCdLvJ/+CicLLgbY4ImOTG+3BegkhNd4advx8busolZu6qcm+ZsefCGS+4bdHYhk9nrc4J2nUCy4G6MCaqpDwzgwcHnlimvk+DquX9wZ+n1W382YRFzNvgOoB8u3sn7PlIaQHCTiXnOzXntm63Bu1AAWXA3xgTd/YM7s/rx0i9AlTyEjY8TJtySWWrf7W8vofMjU8j3WHRbVRnzn9Us2n7I5zU25RwLcKvP8vy9sWFf8K4TSBbcjTEhkZqcSPa4ESx+ZBBPXt+VFmnJ7n2DL2he5gHrmSKl22PT+G6ra92f/XkFFZ5/6IvzKtxf21hwN8aEVLPUZG7pl1GmfNPTw/jwF33LlN/0+kJUleyDYZzbHoW5bGwqpDEmYvQ7pwmDz2/OzPX7S5V3GDPFr+Nzjp2iWWpy5RWrSHHNy4+mGG89d2NMRJlwayZv3X5RtY69+BnfKYgrs2DLgQpfTlIt/VA1GlhwN8ZEnIFdmtG5WcWZIkd0b8mqx8tmqSwoLKrSteZuyuXmCYt49estFdaTKFuNu9LgLiLJIrJYRFaKyFoRecJHndtEJFdEVjhfdwanucaY2uKr+y9n8zPDeOjqstMoP7jzEl65uTcNkhPL7Ju5rmrz0HOOuR7UfrFib7l1PBf1jpakZf6MuRcAV6lqvogkAvNF5CtVXehVb6Kq/jrwTTTG1EYJzlTJewd15vperfnz9I1c3KEx3Vqn0attQ3e9DU8NZfaGHHc2ynv+tYyLOwwGID21ToXXOF1YzLzNrpw3mytYDcpzWOZMUXQMvFca3NX12lfJXSc6X9Fxd8aYmNCuSQov33Shz33JifEM69aCxvWSOHTclZrgomdmApCSFM+sh66gZZorQdnOgydo27iue4jlLzM2luqxq6rP4ZeSB6oljp06Q6qPvxoiiV9j7iISLyIrgBxghqr6Wt78JyKySkQ+EZG25ZxnlIhkiUhWbm5uDZptjDFniQjzfz+wTPmJ00X0e242AOv25jHg+TnumTcH8gtY/33pF5JmVZBaQDweqZb8EolkfgV3VS1S1V5AG+BiEenmVeW/QIaq9gBmAO+Uc57xqpqpqpnp6ek1abcxxpSSkpTA2+XMsskYPZnhL599yWlLzjEyn57J3E2lO5mfrdjj83jvKZB7jpysWWNDoEqzZVT1CDAHGOpVflBVS14fmwD0CUzzjDHGf1d2aca0BwZUWm/wX+f6LJ+86nuf5Ypr0P3ZH3UH4PlpG6vfyBDxZ7ZMuog0dLbrAlcDG7zqeC6/ch2wPpCNNMYYf3VpkcoX91xa7eN/9I8FZQudB6ojerhCXVJ85M8i96eFLYE5IrIKWIJrzP1LEXlSRK5z6tznTJNcCdwH3Bac5hpjTOV6tm3ocwzeH8t3HuHk6dJz5UseqKbVdT1EXbT9ELnHCrjwyems25tX0+YGRaXBXVVXqeqFqtpDVbup6pNO+VhVneRsj1HVrqraU1UHqqr/62UZY0wQtGmUwponrmHNE9eUmjrpS/fWaaU+nz92KqpKTt4p98NT8XpHdeb6/Rw+cYYJ87cFtuEBYrlljDExq34dV4j73BmmKSpW8gsK6fnEdPq0b8TEUX357ccruWdgJwqLlWEvnX3o+sepGyvM3T7mP6uD2/gasuBujKk14uOEtLqJfPqrfnRunkpCfBwvjvQ9f97fRTlOF0bmsnuR/1TAGGMCrE/7xj5TF0wcVTblsLdrujYv9fnLcmbYhJsFd2OMcVzSsQnZ40ZUWGfMsPPLlL2/cAfXv7KAnGOnAJizMYeHPwvvsI0NyxhjjB8eHeEK6hlN67FwzCD6Pnc2vfCjn68BXCmHm9ZP4kC+6yHs/2S2pWclD3ODxYK7McZ4WfTwIBLj42hcL4ndh0+QWieRtJSzwzgt0pLp27ExC7eVXc+1JLADXP/KAtY8cY37wW4o2bCMMcZ4ad4gmcb1kgDXlErPwF7io1H9GDWgY6Xn6vbYNHKOuaZUFhUrhUWheQBrPXdjjKmm3w89DxH45zcVz3X3XiGqsnH9QLCeuzHGVFN8nDBm2Pn8xllQpHXDumFu0VmiYVrxNTMzU7OyssJybWOMCSRVZV/eKVqm1SUr+xD/Wb6Hfy3aWW79Pu0b8emv+lfrWiKyVFUzK6tnwzLGGFNDIuJeECQzozEXtmvEz/u259zmqQx/aR4b95fOG790x+Ggt8mGZYwxJsDi44TzWzYgPk6Y9uAAPv1Vf/eQzdhrL2Drs8OD3gbruRtjTJD1ad+IBaOvCuk1redujDExyIK7McbEIAvuxhgTgyy4G2NMDPJnDdVkEVksIiudpfSe8FGnjohMFJEtIrJIRDKC0VhjjDH+8afnXgBcpao9gV7AUBHxTnp8B3BYVTsBLwB/DGwzjTHGVIU/a6iqquY7HxOdL+/XWq8H3nG2PwEGiYhgjDEmLPwacxeReBFZAeQAM1R1kVeV1sAuAFUtBI4CTXycZ5SIZIlIVm5ubs1abowxplx+vcSkqkVALxFpCHwmIt1UdU1VL6aq44HxACKSKyI7qnoOR1PgQDWPjVZ2z7WD3XPtUJN7bu9PpSq9oaqqR0RkDjAU8Azue4C2wG4RSQDSgIOVnCu9Ktf2JCJZ/iTOiSV2z7WD3XPtEIp79me2TLrTY0dE6gJXAxu8qk0CbnW2bwBma7jSTRpjjPGr594SeEdE4nH9Mvi3qn4pIk8CWao6CXgDeE9EtgCHgJFBa7ExxphKVRrcVXUVcKGP8rEe26eAGwPbtAqND+G1IoXdc+1g91w7BP2ew7ZYhzHGmOCx9APGGBODLLgbY0wMirrgLiJDRWSjk8dmdLjbUxMi8qaI5IjIGo+yxiIyQ0Q2O/82cspFRF527nuViPT2OOZWp/5mEbnV17UigYi0FZE5IrLOyVN0v1Mey/fsMzeTiHRw8jBtcfIyJTnl5eZpEpExTvlGEbkmPHfkP+flx+Ui8qXzOabvWUSyRWS1iKwQkSynLHw/26oaNV9APLAV6AgkASuBC8LdrhrczwCgN7DGo+xPwGhnezTwR2d7OPAVIEBfYJFT3hjY5vzbyNluFO57K+d+WwK9ne1UYBNwQYzfswD1ne1EYJFzL/8GRjrlrwG/crbvBl5ztkcCE53tC5yf9zpAB+f/g/hw318l9/4b4F/Al87nmL5nIBto6lUWtp/tsH9DqvjN6wdM8/g8BhgT7nbV8J4yvIL7RqCls90S2Ohs/xO4ybsecBPwT4/yUvUi+Qv4Atd7E7XinoEUYBlwCa63ExOccvfPNTAN6OdsJzj1xPtn3bNeJH4BbYBZwFXAl849xPo9+wruYfvZjrZhGXcOG8dupyyWNFfV753tfUBzZ7u8e4/K74nzp/eFuHqyMX3P4pWbCVcP9Ii68jBB6faXl6cpqu4ZeBH4HVDsfG5C7N+zAtNFZKmIjHLKwvazbQtkRzBVVRGJubmqIlIf+BR4QFXzxCOBaCzes3rlZgLOC3OTgkpErgVyVHWpiFwZ7vaE0GWqukdEmgEzRKTUm/yh/tmOtp57SQ6bEm2csliyX0RaAjj/5jjl5d17VH1PRCQRV2D/QFX/4xTH9D2XUNUjwBxcQxINxZWHCUq3331vUjpPUzTd86XAdSKSDXyEa2jmJWL7nlHVPc6/Obh+iV9MGH+2oy24LwE6O0/dk3A9fJkU5jYFmmeenltxjUuXlN/iPGXvCxx1/tybBgwRkUbOk/ghTlnEEVcX/Q1gvar+1WNXLN+zr9xM63EF+Rucat737CtP0yRgpDOzpAPQGVgcmruoGlUdo6ptVDUD1/+js1X1ZmL4nkWknoiklmzj+plcQzh/tsP9EKIaDy2G45plsRV4JNztqeG9fAh8D5zBNbZ2B66xxlnAZmAm0NipK8Arzn2vBjI9zvN/wBbn6/Zw31cF93sZrnHJVcAK52t4jN9zD2C5c89rgLFOeUdcgWoL8DFQxylPdj5vcfZ39DjXI873YiMwLNz35uf9X8nZ2TIxe8/Ova10vtaWxKZw/mxb+gFjjIlB0TYsY4wxxg8W3I0xJgZZcDfGmBhkwd0YY2KQBXdjjIlBFtxNrSYiD4hISrjbYUyg2VRIU6s5b1FmquqBcLfFmECynrupNZy3CCc7udXXiMhjQCtgjojMceoMEZHvRGSZiHzs5MEpydX9Jydf92IR6RTOezGmMhbcTW0yFNirqj1VtRuuzIV7gYGqOlBEmgKPAoNVtTeQhSsneYmjqtod+LtzrDERy4K7qU1WA1eLyB9F5HJVPeq1vy+uBSIWOCl6bwXae+z/0OPffkFvrTE1YCl/Ta2hqpuc5cyGA0+LyCyvKgLMUNWbyjtFOdvGRBzruZtaQ0RaASdU9X3geVxLHB7DteQfwELg0pLxdGeM/lyPU/yPx7/fhabVxlSP9dxNbdIdeF5EinFl4vwVruGVqSKy1xl3vw34UETqOMc8iisLKUAjEVkFFOBaDs2YiGVTIY3xg02ZNNHGhmWMMSYGWc/dGGNikPXcjTEmBllwN8aYGGTB3RhjYpAFd2OMiUEW3I0xJgb9f/RfNq84pynVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEWCAYAAABollyxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8VfX9x/HX596bvRchkEDC3mEERQQFREDcCqK1ilbF0W37ax2t1ba2Vm1xVVGL1lXFOsDiBAVFRRRkiAzDJmGEJGSPu76/P84BwpIASW7uzef5eOSRc7/n3HO/3xjf+fI93/M9YoxBKaVU6HIEugJKKaWalwa9UkqFOA16pZQKcRr0SikV4jTolVIqxGnQK6VUiNOgV+oQIrJQRK5vgvN8KyKjmqBKSp0UDXoVNERki4jUikiViOwWkX+LSGyg63U0xpi+xpiFACJyt4i8GOAqqTZKg14Fm/ONMbHAYCAP+N3xvFlEXM1SK6VaMQ16FZSMMYXAu0A/EUkQkZkislNECkXkzyLiBBCRa0TkMxGZLiIlwN0Nyh4TkXIRWSciZx3ts0TkRyKyVkT2isj7ItLZLh8uIsUikmW/zrWP6WW/3iIiY0VkAnAHMMX+18hKEZksIssO+ZxbRWROs/zAVJumQa+Ckh2uE4HlwL8BL9ANGASMAxqOsZ8KbALSgXsblG0EUoE/AG+ISPIRPudCrJC+BEgDFgEvAxhjPgeeBJ4TkSjgReD3xph1Dc9hjHkP+AswyxgTa4zJBd4CckSkd4NDrwKeP4Efh1LfS4NeBZvZIlIGfAp8DPwLK/B/YYypNsYUAdOByxu8Z4cx5lFjjNcYU2uXFQEPGWM8xphZwHrg3CN83k3AX40xa40xXqzAHrivVw/cDSQAXwKFwD8b0whjTD0wC/ghgIj0BbKBuY15v1LHQ4NeBZuLjDGJxpjOxphbsHrpYcBOESmz/wg8CbRr8J7tRzhPoTl4Rb+tQIcjHNcZeLjBuUsBAToCGGM8WP+i6Af83RzfKoHPAT8QEcHqzb9q/wFQqklp0Ktgtx2oB1LtPwCJxph4Y0zfBsccKXw72gG7Tydgx1HOf2ODcycaY6LsYRtEpCPW0M+zwN9FJOIo9TysDsaYLwA3MBL4AfDC9zdVqROjQa+CmjFmJ/ABVsjGi4hDRLqKyJnHeGs74GciEiYik4HewDtHOG4GcLs9tIJ94XeyvS1YvfmZwHXATuBPR/m83UC2iBz6/9zzwGOAxxjz6THqrNQJ0aBXoeBqIBxYA+wFXgMyjvGeJUB3oBjrAu0kY0zJoQcZY94E/ga8IiIVwGrgHHv3z7D+YPzeHrK5FrhWREYe4fP+a38vEZGvG5S/gDXso3PsVbMRffCIamtE5BrgemPMiFZQlyisC8ODjTH5ga6PCk3ao1cqsG4GvtKQV81J7xJUKkBEZAvWDJ6LAlwVFeJ06EYppUKcDt0opVSIaxVDN6mpqSY7OzvQ1VBKqaCybNmyYmNM2rGOaxVBn52dzdKlSwNdDaWUCioisrUxx+nQjVJKhTgNeqWUCnEa9EopFeJaxRi9UkqdCI/HQ0FBAXV1dYGuSrOKjIwkMzOTsLCwE3q/Br1SKmgVFBQQFxdHdnY2By9GGjqMMZSUlFBQUEBOTs4JnUOHbpRSQauuro6UlJSQDXkAESElJeWk/tWiQa+UCmqhHPL7nGwbgzrov9pSygPvr8Pn12UclFLqaII66FdsK+OfCzZS4/YGuipKqTaorKyMxx9//LjfN3HiRMrKypqhRkcW1EEfHeEEoNbtC3BNlFJt0dGC3uv9/s7nO++8Q2JiYnNV6zBBHfRp7gLOcyympl579EqplnfbbbexceNGBg4cyNChQxk5ciQXXHABffr0AeCiiy5iyJAh9O3bl6eeemr/+7KzsykuLmbLli307t2bG264gb59+zJu3Dhqa2ubvJ5BPb2y854FPBb+KGurb4K02EBXRykVQPf871vW7Kho0nP26RDPH87ve9T99913H6tXr2bFihUsXLiQc889l9WrV++fBvnMM8+QnJxMbW0tQ4cO5dJLLyUlJeWgc+Tn5/Pyyy/z9NNPc9lll/H666/zwx/+sEnbEdQ9emLaAeAt3xngiiilFJxyyikHzXV/5JFHyM3NZdiwYWzfvp38/MMfJJaTk8PAgQMBGDJkCFu2bGnyegV1j17i2wPgq9gd4JoopQLt+3reLSUmJmb/9sKFC5k/fz6LFy8mOjqaUaNGHXEufERExP5tp9PZLEM3Qd2jd8ZnWBtVuwJbEaVUmxQXF0dlZeUR95WXl5OUlER0dDTr1q3jiy++aOHaHRDUPfrwRCvopVJ79EqplpeSksLpp59Ov379iIqKIj09ff++CRMmMGPGDHr37k3Pnj0ZNmxYwOoZ1EEfGZdKvQnDVVsU6Koopdqo//znP0csj4iI4N133z3ivn3j8KmpqaxevXp/+a9//esmrx8E+dBNdISLPSQQXqNBr5RSRxPUQR8V5qTIJBJeV3zyJ9u6GLZ+fvLnUUqpViaoh24cDqFEkshyN0HQf/Qn8Hvhug9O/lxKKdWKBHWPHmCvI5mYpgh6dxXUlZ/8eZRSqpUJ+qAvd6UQ7asAb/3Jnchdo0GvlApJQR/0Va5ke+M4LshuXQxbPj24zFMLdU17+7RSSrUGjQ56EXGKyHIRmWu/zhGRJSKyQURmiUi4XR5hv95g789unqpbqsNTrY2q45hLv+BemH/3wWWeGvBUg08XSFNKnZi7776bBx98MNDVOMzx9Oh/Dqxt8PpvwHRjTDdgL3CdXX4dsNcun24f12xqIqz1bqjY0fg31VdC7SFrQXvs247rtVevlAotjQp6EckEzgX+Zb8WYAzwmn3Ic8BF9vaF9mvs/WdJMz7rqzrSXgahfPvBOzzf83xFzyHj8X4/eO2g13F6pdRxuPfee+nRowcjRoxg/fr1AGzcuJEJEyYwZMgQRo4cybp16ygvL6dz5874/X4AqqurycrKwuPxNHsdGzu98iHgN0Cc/ToFKDPG7BvnKAA62tsdge0AxhiviJTbxx80NUZEpgHTADp16nSi9YeoRGqIJLqsQdDv+Q6eOA1uWAAZAw5/z6EXXr0NFhHSHr1Swend22DXN017zvb94Zz7jrp72bJlvPLKK6xYsQKv18vgwYMZMmQI06ZNY8aMGXTv3p0lS5Zwyy238NFHHzFw4EA+/vhjRo8ezdy5cxk/fjxhYWFNW+cjOGbQi8h5QJExZpmIjGqqDzbGPAU8BZCXl3fCD32NjnCxkzS6lm8/ML5euMyaE1+66bCgr6jzEOepRnz1Vq8/LPLAsA3oBVmlVKMtWrSIiy++mOjoaAAuuOAC6urq+Pzzz5k8efL+4+rrrVmBU6ZMYdasWYwePZpXXnmFW265pUXq2Zge/enABSIyEYgE4oGHgUQRcdm9+kyg0D6+EMgCCkTEBSQAJU1ec1t0uItCk2oF/eyboa7M+isM1lh8A8YYxjz4MV/4a6yG15VbQe+uPnCQDt0oFZy+p+fdkvx+P4mJiaxYseKwfRdccAF33HEHpaWlLFu2jDFjxrRInY45Rm+Mud0Yk2mMyQYuBz4yxlwJLAAm2YdNBebY22/Zr7H3f2SMOeEe+7FEhzvZ7k/GlG2HjR/B5kVQtM7aeUjQV9R5Ka2qxeW359zvC3WPDt0opY7fGWecwezZs6mtraWyspL//e9/REdHk5OTw3//+1/A6mCuXLkSgNjYWIYOHcrPf/5zzjvvPJxOZ4vU82Tm0f8WuFVENmCNwc+0y2cCKXb5rcBtJ1fF7xcd7qLAn4bUlkJNsTXevmmBtfOQoC+tdhNFgxur9gd9TYOyIwS9MbBtifVdKaVsgwcPZsqUKeTm5nLOOecwdOhQAF566SVmzpxJbm4uffv2Zc6cOfvfM2XKFF588UWmTJnSYvU8rrVujDELgYX29ibglCMcUwdMPrS8uUSHO1lrUg8u3Bfch/TOS6vriT5i0B+jR79tMTx7Dlz/IWTmNUGtlVKh4s477+TOO+88rPy999474vGTJk2iGQc5jijo74yNCndSsC/oneEQHndg5yE9+pIqN1HSMOjtufQH9eiPMEZfuevg70opFUSCPuhjwl3sMPZT1dP7QYeBB3Ye0jvfW+M+So/+GEG/7zx6oVYpFYSCPuijw50UkYTPFQWZQ6HDIGtHYqfDe/TVbqJpcCPVoUM3zogjD93sO66u7PB9SqmAaulhkEA42TYG9Xr0YAW9Hwerz/4Puf0Hgs8NGbnw9fOHX4w9bOjGDvB90yvj0o/ca99XduiyCUqpgIqMjKSkpISUlBSa8Qb8gDLGUFJSQmRk5AmfI+iDPi0uAoBv6UZutL2SZf9JsGY2lOyBt39lBfnFMyg96tCN3aOPyzjyrJv9PXodulGqNcnMzKSgoIA9e/YEuirNKjIykszMzBN+f9AHfU5qDBkJkSzK38MPTm2wlEJEvNWj37bEWpUSa3plvB30RhzIIUHvjkrDv+tb9v/d9NlrUOwLfx26UapVCQsLIycnJ9DVaPWCPuhFhDO6p/HO6p14fX5cTvuyQ0ScNd7urbN69MZQWu2mvT1044tuh6vhxVhHGOvKw8goL0W8PiLqSih5fDx7Y7rQLdG+qUF79EqpIBT0F2MBzuiRRmWdl5UFB3rcXlcM/roKTHWxFeT1lZRUuUl0ugFwR6fjbzi9MjyaIk8k8dRQVFYDL1xCSs0mXCXfHXmMvqYU3rvj4OUTlFKqFQqJoB/RLRWHwLOfbcHvN/j8htlrK3FgEOyr1ZW72FvjJiPaep1fE8vWwp0UVdZZQR8WzTpPOhHigc8fgd3fsNskkeDf22B6ZYOg//Jp+OKf1s1USinVioVE0CdEh/HLsT2Yu2on9723jiWbSvh6t++gY9xlhdS4faRH+fAYJ/nVkcSaah6an2+N0YdF8XrtYOpNGB2X/wN/WAyv+UaSRCX+KvtCz76evd8PK160tit2tmBLlVLq+IVE0AP8ZEw3JvZvz2vLClhZUE6ViTpof1WJtbhmaoSXWiIo9kYRTzXvfrmGz9Zuo8IXxuYqFx/4h+AwXvZ2Hs82kw6Ao8YO+n1DN5s/hrJt1vbxPNlKKaUCIGSCXkQY0S2N0mo3b3+zg4jYhIP2VxZZDyZJDvNSQwRf+7sTIV4WR99KR+82CqqsObizfKMBWN/+fPaYBudwRlgLpnnrIf8DcEVBVBJUFKKUUq1ZyAQ9wKBOiQCsLqygfVoaAD4j+JxRfLZiNSkx4WREG2pMBB/4hzI371kifVVks5NStzWz5lN/f27rPIuVrlyKTOKBkydmWd/ryqFkA6R0g8TO2qNXSrV6IRX0PdLjiI2wZox2bN8egBISKPQlkugr4ZVpw4iinnqxZsrHdD0dIq0wr8W68apbu1jW1sRRsLeG+ogGq2Im2nP0a8vsoO8K8R016JVSrV5IBb3TIeRmWcMtXTKtoN9tEtnhT6RnTDXd0+PAXY3bYQV9l3ax1nIJQB3huBxCbmYiu8prKdhbS0xyBn7s26r3BX31Hti71erRx3fQoRulVKsXUkEPkNc5GadD6JHVAYBSRwq7TRIZjgNz5j3OKMKcQsfEqP1B73ZE0SExio5JURRV1rOlpJqM5DgqHfHW+xI7W993rgTjOxD0dWU6l14p1aqFXNBPO6MLr988nMQka92busg0yp3JRNXtsZ4Q5a7B54yic0qMdRetHfQpSYkMyEwgIyESY2BrSQ0dE6OoDrOHb/b16AuXWd9TullDN6DDN0qpVi3kgj4mwsXArERwhkHXMWQNnsCgvOGItxa2fQGearLSU/j5Wd2tN9jLGp/ZpxOPXjGIDonWtMz28ZFcNjQLT5R1Ubc8IsM6vnApAItKE6wePcBTo+Gd37RcI5VS6jiEXNAf5Ko36TPuWvqPu9a66LrkCXDX0CEthfNz7ZBOyoGcM5CsofYUzVQenJzL+784gx7pcSS2s2bbvFMQAQ4X7N1CjTOBq17OZ5fYDzxxV8KXTwaokUop9f1CO+j3CY+GIVNh7VyoLYWwmAP7HA6Y+j/ofT5gXdCdNCSThOgwABLSO+PBxSurq2HCfYCw3WktFzpnox9c1oXdypjslmyRUko1WtsIeoBht0DHweD3QfJxLGs67GbmDZnByh1VbMi+Aq6bx5+YBsBrK0swN33Gu66z8NfsbaaKK6XUyQn6ZYobLa49XD8fPHUQdhxPaolJpe/wifDZQpZsLqHjoMF8WlFEx8Qo8ouqWO9tx7b6WM52VlkXe0P0KTdKqeDVdnr0+xxPyNs6JUeTHBPOim1lbC62plJeOsQavvl4/R5K/NG48IG7qkmrqpRSTaHtBf0JEBFyMxNYsb2MjXusMD+rVzscAgvX76GcWADqq0oDWU2llDoiDfpGGpiVxIY9VawqKEPEWm6hc0oMS7eWUmasi7vlJUUBrqVSSh1Og76RBnVKxBh4a+UOOiZGERXupGtaDB6foQIr6KvKigNcS6WUOpwGfSPlZiXidAi7K+rJzbQWQuuaZg3ZlBnre01FaD+JXikVnNrOrJuTlBAVxqxpwwDon2ktnLYv6J0xSeCFugodo1dKtT4a9MchLzv5oNdd21lDNtmZmbAFvNUa9Eqp1keHbk7Cvh59lw7t8Bin3jSllGqVtEd/EhKjw3n66jwGdUqk8vNYa8lipZRqZTToT9LZfawHiBc443DWa9ArpVofHbppInWueMLdFYGuhlJKHUaDvol4wuKJ8mnQK6VaHw36JuKLTCTGX4UxJtBVUUqpg2jQNxGJTCSBKirqvIGuilJKHUSDvok4Y5KIpZaSytpAV0UppQ6iQd9EwmMScYihrExvmlJKtS4a9E0kMta6a7a8VBc2U0q1LscMehGJFJEvRWSliHwrIvfY5TkiskRENojILBEJt8sj7Ncb7P3ZzduE1iEm3gr6qkq9O1Yp1bo0pkdfD4wxxuQCA4EJIjIM+Bsw3RjTDdgLXGcffx2w1y6fbh8X8mISUwCoqygJcE2UUupgxwx6Y9n3jLww+8sAY4DX7PLngIvs7Qvt19j7zxIJ/QepuqKsFS3rq/TuWKVU69KoMXoRcYrICqAImAdsBMqMMfvmEhYAHe3tjsB2AHt/OZByhHNOE5GlIrJ0z54QWMc90gp6ry5sppRqZRoV9MYYnzFmIJAJnAL0OtkPNsY8ZYzJM8bkpaWlnezpAs8Oel+t3h2rlGpdjmvWjTGmDFgAnAYkisi+RdEygUJ7uxDIArD3JwChP3AdEQ+A1JUHuCJKKXWwxsy6SRORRHs7CjgbWIsV+JPsw6YCc+ztt+zX2Ps/Mm1hXQBXOG6JwKELmymlWpnGLFOcATwnIk6sPwyvGmPmisga4BUR+TOwHJhpHz8TeEFENgClwOXNUO9WyeOKJaK2CrfXT7hLb1FQSrUOxwx6Y8wqYNARyjdhjdcfWl4HTG6S2gUZb3g8cXU17K1xkx4fGejqKKUUoHfGNikTEU88NeyprA90VZRSaj8N+ibkjEogXmrYUaYLmymlWg8N+iYUEZtEHDV4NyyAWp1Pr5RqHTTom1BYTCKZjmImLr8Jlj4b6OoopRSgQd+kJDKBCDzWi8qdga2MUkrZNOibUmT8ge2qosDVQymlGtCgb0oRCfs3jQa9UqqV0KBvSpEHgt5bqUGvlGodNOibUrS1SOcmf3ukWoNeKdU6aNA3pS6jKB4/gzd9I3C5K8BTF+gaKaWUBn2TcrpIOGUKxZJkva4OgXX2lVJBT4O+iYU5HZhoe319Hb5RSrUCGvTNIDyxvbVRpT16pVTgadA3g5jkDtaG9uiVUq2ABn0zSG5nPT63vmxXgGuilFIa9M2iY1oSFSaK6uLt4NGVLJVSgaVB3ww6JUezxySSvOZ5eGwotIEnKSqlWi8N+mbQKSWat3zDKY/KgvLtUFcW6CoppdowDfpmEB8ZxnMRl/N2u2lWQXlBYCuklGrTNOibSU5qDN9U2qtZatArpQJIg76ZnJKdzMe7I6wXGvRKqQDSoG8mp3VNYacvFr8jzBqnV0qpANGgbyan5CTjcjopD0vXHr1SKqA06JtJdLiLQVlJbPMla9ArpQJKg74ZjeyeSn59Ir69OnSjlAocDfpmdE7/DApNClK1C3yeQFdHKdVGadA3o27tYjFxHXHgh4rCQFdHKdVGadA3s/a9h+E3wnePXspXq9dBTSnsWBHoaiml2hAN+mZ21uhxzMz6C118m6j8+DH48I/w73PB7w901ZRSbYQGfTNLi4vghutvYVtEd1JLl8GWT8FdBVW6hLFSqmVo0LeQinZD6eVdDyX5VkHp5sBWSCnVZmjQt5CY7iMJF9+Bgr0a9EqplqFB30KyckcD4HFEgDhg7xYA/vHBev6zZFsAa6aUCnUa9C0kMjGd7a5slple+OMzoXQzmxfP4eWPljLz002Brp5SKoRp0Leg8otf4Ke109jgTaP+uw/Jef9qZkf8HileT2m1O9DVU0qFKA36FtSv7wDG5A1gaXkCEe691BsXCS4vd7leYNnWvYGunlIqRGnQt7D7Lu3PuBGnWi96TiRy0GUMcXzHss1FBw7auhh2fxuYCiqlQo4GfQsTEVKzBwAQMfRqXNmnESP17Mlfah3g98OrV8P7dwSwlkqpUKJBHwg9xsO170K3sdDpNAAS9izjnws2QNG3UF2kPXqlVJM5ZtCLSJaILBCRNSLyrYj83C5PFpF5IpJvf0+yy0VEHhGRDSKySkQGN3cjgo7DCZ2HgwjEd8AkdubixI08+f7XFK98zzqmeg9U7QlsPZVSIaExPXov8CtjTB9gGPBjEekD3AZ8aIzpDnxovwY4B+huf00DnmjyWocY6Tyc/tWf83XEjcQufwrEae0o0l69UurkHTPojTE7jTFf29uVwFqgI3Ah8Jx92HPARfb2hcDzxvIFkCgiGU1e81Ay+k5qz76PL/y9iawrgn6XWOW71wS2XkqpkHBcY/Qikg0MApYA6caYnfauXUC6vd0RaPhIpQK77NBzTRORpSKydM+eNj5EkZhF1Ok3c6vrDuZlTIOx90B0qvbolVJNotFBLyKxwOvAL4wxFQ33GWMMYI7ng40xTxlj8owxeWlpacfz1pDVPiWRF8ImQ0JHdkd1pejbjzHr3gafN9BVU0oFsUYFvYiEYYX8S8aYN+zi3fuGZOzv+yaCFwJZDd6eaZepY8hKjmZ7aQ0lVfW8UZxFO/d25JUfwNOj9q+No5RSx6sxs24EmAmsNcb8o8Gut4Cp9vZUYE6D8qvt2TfDgPIGQzzqe3RKjqZgbw3T53/Hg55LGO17jJey7oaSTfDRvYGunlIqSDWmR386cBUwRkRW2F8TgfuAs0UkHxhrvwZ4B9gEbACeBm5p+mqHpk7J0Xh8hpeWbOOyvE6ckpvLvVt7U9P/Svj2DSjXfxgppY6f61gHGGM+BeQou886wvEG+PFJ1qtN6pQcvX972hldqPf6eGvlDn60dggvm38hTwyHjoPhqjcDWEulVLDRO2NbkX1BP7Z3OjmpMfRqH8+L15/K+vpk/uK+nB3OjrDxIygvCHBNlVLBRIO+FclMiuKnY7px2zm99pcN6ZzEot+OYWP3H/GLavuSyJbPAlRDpVQw0qBvRUSEX43rSde02IPKYyNcXHFKJ5bWdcATFg9bPw1QDZVSwUiDPkiM7J5KdEQ430X2hy0a9EqpxtOgDxKRYU7G9m7Hu1VdoXQTfPMamOO6R00p1UZp0AeRc/pn8FLtcCqSB8Dr12G+fCrQVVJKBQEN+iByZo803OFJ/Mh5L5/4c/HOuwcqGnkv2qcPwYuTmreCSqlWSYM+iESGOTmrdzpLt1fyO8814HXD9L4HArx0M3hqj/zm796HTQvB72up6iqlWgkN+iBzWV4WMeFO2mf35lrfHfi6jYMN82DbEngsDx7sAatfP/hNxlgrYfo9ULkrMBVXSgWMBn2QGdE9lW/uHs/No7ryqacnK7reaO344Hfg90J4DCw5ZOy+YgfUlVvb5dtRSrUtGvRByOEQTuuSQoTLwc3z3NQ4YqHgS0jpBr3Ph13fWA8Z36eowQNMyjTolWprNOiDVGSYk99M6EWvjkl86ukJwK70M/GmDwBPNZRu3H+sd+fq/dumbFuL11UpFVga9EHsuhE5PP+jU6jIGA7ArSvS+dcG+67anSv3H7fzu2XsNMmUmDjqi7cEoKZKqUDSoA8B51z1fywZ8CdS+41l+goHfkcY7Fi+/2El/l3fsN5kUWhS8ZRuDWxllVItToM+BMTEJXDqJT/jTxcPIC4mmu2ubFj8T3h4IObrF+js3cK2hFMoNKnIvouxu77RGThKtREa9CEkISqM8wZ04MO6HpiwKIiIg7m/wGOc1PS8hAKTRmT1Dmsq5tNj4J1fB7rKSqkWoEEfYsb1Sec+92XMn7gITvsJ4veywD+Q7Owu7HG0w+WvgxcuAp8bNn2sDx5Xqg3QoA8xQ3OSiYqK5vmvS3g/9nxKUwbzlPdcspKjWJcwnK9iRkG/S+Gsu6C+AnZ8ffhJtnwKj58G1cUtXn+lVNPToA8xYU4H4/qksyi/mBtf28zdqf9gqelFZlI0zuQc7o74NVz4GAy5FhCYdxf8vTc80A02L8IYg++b16y598v+HejmKKWagAZ9CLrnwr68eN2pAHywZhdxkS4SosLokBhFYVkttW4fV72cz564XrBtMSRkgt+L+/MnuOzJxexc/oF1oqXPgM8TwJYopZqCBn0Iig53MaJ7KhkJkdR5/GQmWc+i7ZgURVmNh2kvLGVRfjGPei6EYbfANXMh9wdI/vtUbF1Jpr+QktRToKKQwvn/DHBrlFInS4M+hA3unARYz6IFGJiViMshfLahmNysRJ4vG0DBqb8HVwRFXS8hDC//TnkBgBt2XcgCXy7tv/gj5M+zTmgMbP4EvPUBaY9S6sRo0IewIZ0ODvrhXVNZ+6cJrPnjBB6cNACARfnWBdeFZenM9w0io+pbqh1xrPB25meen7LDlQUvTYL5d8O3b8Bz58NHfwpIe5RSJ8YV6Aqo5jPE7tF3TIzaXxbmdBDmhG7tYslIiOS1ZQUkRYfxcf4evoy4k7OujKBor4duq0ojAAAeKElEQVRx3ybicgqXfvcnlgz6APl0Ov7wWByA/4sncQy9HpKyA9MwpdRx0R59COvfMYHfndubiwZ1PGyfiDChX3uWbd3LTS9+zdurdjKyexqScwY5g89ixlVDGNEtlaI6J1tO+wt0G4vDXcVvPTfgMQKfPRyAFimlToQGfQhzOITrR3YhNTbiiPvvOq8PX905lrvO60OYU5jYP+Og/blZiQB8uL6YPec8xe0J9zHLN4oPvbl4176jDydXKkiIaQX/s+bl5ZmlS5cGuhptmtvrJ9x18N99r89P/7s/oNbjI9zlwO31c8PIHMoWP8cDrhlw6Uzr5qrYdnDaTyAy3lpI7cM/Qdk2OP9hSO8TmAYp1QaIyDJjTN6xjtMxegVwWMgDuJwOpk/JZXdFPYs3lvD5xmJuGNmFewpH49/xJI7XrwdnuLWcQn0lTPgrvPMb2LLIOsEn98Pkf7dsQ5RSh9GgV99rQj9rOGfq8Gx8foPTIQzp05PlBd0Y7NyIXPUGrPiPdXNV5lDIf99aXqG2DBY/Bnu3QlJn62TeenAdeRjpqHwe618HKV2buGVKtR06Rq8azekQAEb1TONuz1QWDvwHZI+Akb+yevWvXQsxaXDKjXDqTSAOeHQwvDENivPhr1mQP//4PnTBvfD4MKjc3QwtUoE2Z0UhLy3RZyQ0Nw16ddxyUmOoSO7HE7t6UVxVz+xtkdRe/Ayc+3d2Tp7Lr+ZsZMKzG9kw/gXqO4/CrHoVlswAXz188U/w+2DJk/Daj77/5qv6KvjqGeuPyLq5LddA1WL+u7SAl77Qx1s2Nx26UcdNRLhqWGf+/PZa8v5s9dBH9+zII1ecxzVPLGb73hqcDuFXX8UTV3E+LzIf89VMBGDjRzDzbChcZp2s+zjwuTHr30Fi20OfC2Dt/+DM22DNHKgvh8gEa3vodfDNa9Zia9EpMOkZcDgD9WNQTaDe66PO6wt0NUKeBr06IdeP7EKP9DgWrt9DTISTRz/awJA/z8fj8/Pctaewo6yW2974BoekkR/Wke6OQhj2Y/jySShaCxc+Dp89jP+923HUlrLdpNNBSnAte9b6gPAYWDuXuvZD2BAzmL6bnkGK1sJbP7UuANeVwe5fQcaAwP4g1Emp9/qp9/gDXY2Qp0GvTtgZPdI4o0caAFlJ0Xy3u5JTu6RwRo80vD4/89cWMbxrCptWnE334n/zTtS5jLtyPK6Ejjy73snu0hXc5n+KL/29+G/fx1m54isuTtxAn5qlnLH4ccT4+KPrUpZuTeP9SAP/GmsN9fxglrUUw5ZFsP5dSO8LIrBhPoy6A2LTAvyTUY3l9vqp82iPvrlp0KsmcdnQrINeu5wO/jXVmt5b1v+P3PXCAJ5/twwR6N1+J2t2VtA/fTyPVLnpd94NPJDXj4dS4nj4484MJ40zZTl7nOm87RlCnVN4pfM9XLHtjzDwCsg5A5JyrHH/skPGd9fOhRs+hMROLdV0dRLqNehbhAa9anaJCQncdfO1DF61kw1FVXySv4fRPdN44odDiAwbu/+4X4ztwU/HdOfuOR147+t5vOM+hV+c25uV28v4y1oHbyc8xdjkfkR8uY2k6m5McM+zxu+HXAvGB30uhplj4evnYczvAthi1Vj1Hh91Xh26aW4a9KpFuJyO/Wvu/Hp8z6Me53QIl+Z15qIlvyQjIZL7T+lEz/ZxzF6xg6/Lolny3gacDmGCrzsTwucxL2oCWyOnMmlIJisLyhncYQRxq2ZB3nVQuQNi20NFoTV/P70/OHSiWWtS7/Xj8xs8Pj9hTv1v01w06FWrk5uZwOVDsxjdqx2RYU6Gd03lw1+dSUJUGBMe+oR6r58ljlOY48znTztHUfz2Wv723jo8PsPFjr5MD/8EHs61pnM21HMiXPwkeGogrn1gGqcOMtS7DHHUUOcZp0HfjI651o2IPAOcBxQZY/rZZcnALCAb2AJcZozZKyICPAxMBGqAa4wxR3j69MF0rRvVWJuLq3F7/cz4eCNvLi+kU3I0k4ZkUlRZx8T+Gfzj7a95uvQ6EjJ74hh2C99u2MIDX9bwk54V5G1+Eh8OcEXhvHU1LHsW0npZfwBEwO+HtW9B5+HW+j2q2X121+kkUkm7//uKtLjjvGtaNelaN/8GHgOeb1B2G/ChMeY+EbnNfv1b4Bygu/11KvCE/V2pJpGTGgPAOf3a8+byQq4bkcPU4dn79/vOGcSImdP5YYfe3Na3DzO+WcFC/w4WrReulhoGurZxofdjSl/8Eck7FlhvGnQVjP8LvH4d5H8AnU6Da97RYZ5m5vMbInATTZ1ekG1mx/xNNsZ8ApQeUnwh8Jy9/RxwUYPy543lCyBRRDJQqomN7Z3OzKl5XHnqwbNrRnRL5by8Hjz5yRb+NHctH63dzcjuqTgdwvYeUxl7xxusdfYgeccCfHGZ1qqby1/AN+MM2PAhe7LPtx6Y/uVTB0669n+6BEMzcHv9RFFPjNRTrzdNNasTHaNPN8bstLd3Aen2dkdge4PjCuyynRxCRKYB0wA6ddKpcOr4OBzCWb3TDysXEf56SX8iwhw889lmwLq564FJuaTGhuNyOkg+Yxos+DVvRE8iqv31ZDk+JLdsLfN63M0Nq7rzcnQhp87/A44e48Drhlk/hM6nw9S52stvQvVeH5G4iaKe3W6dedOcTvpirDHGiMhxL2pvjHkKeAqsMfqTrYdS+zgcwj0X9CUhKozPNhRzWpeUg5ZhTh9xLf8r9HDbqkx8W1cxNP0PJNZsZt6qzvRIj+UPFdN403cr0bNvQToMtt609TP46l9w6rQAtSr0uL1+oqSeGOqo83gDXZ2QdqJBv1tEMowxO+2hmSK7vBBoeOdMpl2mVIsSEX41rie/GneEqZxOF2dPvpFR7q8Z1CmRm87syne7q/B9sJ7fn9eHzzYU87u3pjJ92xOwbTGV2eOJc/lh/h+g+9nWzVj71thZ/x5s/BDOud+6oKsard7rJxY3DjG466qBlEBXKWSd6L9D3wKm2ttTgTkNyq8WyzCgvMEQj1KtRmSYk5nXDOUnY7rjcjro0yGeZ64ZSk5qDJcOzmRB5Bie9Y4H4Nb8/nzQ7Q5wuODJM+HeDNi8COrK4a2fWOP5Gz8KcIuCT73XRxTWFFhPbVWAaxPajhn0IvIysBjoKSIFInIdcB9wtojkA2Pt1wDvAJuADcDTwC3NUmulmlFUuJMbRnbh8fAfsWzCHIo6jOH2eSVUT3gIMvMgLh1m3wyzb4HqYohKhkX/OPgkPq+1Bn8reFRna1Xn9hIpHgB8dZUBrs1xKloLK2cFuhaNdsyhG2PMFUfZddYRjjXAj0+2UkoF2i2junLjGV1wOR3cm1XO+Y99yo3LMhnfbzrJKSs556trMOU7eCfxStq378DQdfdb6+x0GQXr3oZPp8OetdZF3IufhMQsax3+sm2QnBPo5rUKnrqa/dtBF/RLn4XlL0DulEDXpFH0zliljkBEcDmtMfd+HRO4/ZxePL5wI59uKAacZMv91IUlEeVNoXBFGZ+k9qb97Jutp2rVleFP6sLOAT+lw7pnkVd+AKNug4/+DEVr4Lr5kDU0sA1sBTz1B4Zr/PXVAazJCXBXW3dY+7zgbP0x2vprqFQrMO2Mrvzo9Bz2VNXjdAhbS2rokR5HfKSL376+iknLbmRuzL3EZQ9iY4/rmLYwjC1f1jE+LJIndz0Ar/wAkrtAeKz14JRDg96Ywy/m+v0tO53TGOsh75HxLfJx3voDPXoTZEG/ffcea9aJpxqcCYGuzjHppGClGsnldJCREEW7uEiGZieTEBWGiPCXi/tz0ajhDKp+mJ9wG9cvjMDjFx6cnEvcgPP5tedG7vFfz+87zsTd6yL49k0rUPepLoEHulqrbu5Tth3+lg3LX2q5Bq6ZDX/vCTWH3h/ZPHwNhm6MO7guxlZXWf/9TH1wDDlp0Ct1klxOB78e35Nfj+vJu6t3sa20hgcmD2DSkEwenJzL1TffQfWAq/nPsl3csqYveKrxvT4N9m7B6/OzbdELUFMC8/4AtXutky560HqM4kd/BnfN9z9bt6kULLWGI0o3N/9nAf4GQze4g6tH7/RZf6TcNRUBrknj6NCNUk3kpjO7sm5XJe3jIxjeNXV/+YDMRO6flMgPh3Xmd2/Gcf+uy/jp+tmE5c9jcbvLidv5GTGuVJJrS5DnLoCOg2H5i9SlDyZy99eYB7sj4bFw9Wxo17v5GlCywfpevg0yhzTf59j87gM9+mALepevFoDaqnKCYSk2DXqlmojTITx6xaCj7h+Qmcicn4xgUX4v7vnsUvI2/ZNJu18AB9znuZL+OR2Y6JmHWTuXuuQ+jNl2HT93JpDtqmYY25B/nwvXz7fG+pvDvqAv2/79xzWRhkHv8AZX0If56gCoq9YevVLqECLCGT3SGNl9PI9+1JVn173K1IgFlIVdzO+3wVvZE/h8ZwndYmOpjazGN/5hrpi9mt/0dnHTdzfgeHUqXD0HopObtmI+D+zdYm2Xt0zQG0/t/m2Hp+Z7jmx9wv1W3euDJOh1jF6pABARfnZWd6798Z04rp/PuFMHUFrt5v1vd1Pn8bF8WxnXj8jhh8M6c35uB+7/ysuNVTfArlVwfxf4S0d4/iIroBuqLrHm8x/vjVp7t4LfWm/GU7rtGAc3kQY9emeQ9ejDjdWj99QGR9Brj16pVuCM7mm0i4sgPiqMh6YM5L9Lt3PtCOvGqoemDGTqaZ3554I0zv0uiZs6bGBMu2pi1r5qLbQ2YAqsmgVVu2HVq9ajEy95GgZc1vgKlOQDUGziiSrZSlhzNPJQnoZBX/s9B7Y+kfuCXi/GKqUay+V08PK0YcSEu2ifEEm/jgfmZjsdQl52Mk9fncdjCxL5v4+7ErPXyYLM3cS/fyfmvdsRDF6c7HZmkJHaC8e8P0CvcyE8plGfX7xlNanAZ/5+TKj8pplaeQh76KZS4gjzBdHQjd9PJG5rU6dXKqWOR9e0WNonRB51v8vp4Bdje/D2z0YSHeni3I2X8EHEOB7yXsL4+vvI9f+HkTV/4+8Rt1gPRv/g99Z0zTk/hns7wEuXQcWOI567cMMKSk0cG505RHgroa75e6pi9+IrnImE+YKoR+9puHRDcMz/1x69UkGma1osc348gsc+2sD9+TmMG5HOLzMTyctO4s2vC7n3nbWM6XoVQ5bOhFWzMJ4a1iaNpvemhcjCv8IFjx58Qk8tXYsX8E10HhHRnaAEKC+AyD7N2g6HtxYfDmqc8YT7g6dH76+vPtBDDpIevQa9UkEoOSacu84/PIivH5nD2p0VTFl+Nu9nFlDrhd8UjWPNjs48FOvkgpWzcORdZy2rvGc9DP8ptdtXEGuqKeh6BTE+B5SAr3gDzvTmDvo66gnH7YwmwlverJ/VlOpqKom2tyVI7ujVoFcqhIgIf7mkP7UeH2etvgERmHpaNnf0TuehWXu4yPcBPHUmAD5nFLLqVVyOcDb6M8gYMJbivaXsXpNI0rx7cHY/q9Fj/CfC6aulTiLxOqOJ8+9qts9pagcHfXDMFtKgVyrERIY5efzKwcxbs5v2CZEMyEwEoNPNl/HPGV/j8NZS1u1iZn1TwbWu9+kiO5jtH8nDnZPYFhvBLz238FLZX+G92w4f5mlCTl8d9RKB1xW9fxZLMKivPTBcEyzTQvVirFIhSEQY17f9/pAH6JQSzfk33cdM5xSe/MYweeQA4ibcxU89P2N3+zOJjXDROyMOV7fRPO2/wFpk7ds3m62OLl8dbonE74oi2tTCpo+tNftbOXeNNVzjMU5cQRL02qNXqg3plBLNGzcPp6iyjrzsZDw+P3NX7WBsn3TA+gNx3yX9OXf6FEaHrafbG9OQyt3WnHxvHexcBeHRkD3ypJ+R6/LX4XFE4HPFkChV8PwFcNnz0OfCpmhqs/HYD0kpJiFopoVq0CvVxnRKiaZTijXKHOZ08MYtpx+0v0NiFLefN4BJr9/KG+nP0PW938J7vz34JNkj4dJ/QVx763VVEayZA51Og/b9GlWPMH8dHkckETS4u3fTx60/6O3n25aQSEqQzBbSoFdKHWZyXibz13Zl7JqfMFTOZrAjnyqi2BWRw0UdyjinYAbOp0ZD9gjoezF88ThsWWS9+fSfQ//LICwKUroe9TPC/HW4XUl4+k3hP9uL6B2xh/6bPmn1oeSrs4ZrKl1JdPSXBLg2jdPaf6ZKqQAQEWb8cAhbS2soqxmB3xgKy+p4Z9VO/u+7PTzhTeFf0bOIWTuf+G9eBWC6XM2V3dy0++xh+OxhjDiQfpOsIZ7ModbwT+SBO37DTT01zkhOOX0MNan9eO+FuxhU+jJU7jrwL4VWyGevo18bnkJk7eoA16ZxNOiVUkfkcAg5qTGANcVySGe4ILcDbq+fW15K4bS1OUQ7vPzS8QpheHkp4gIeXV3PJY4EnPgZ6NjAJWveJjwqDlk1C5a/CNe8DRGxAESYerxO607gUT3b8XHm6bDrZXwbF+IceHnjK1q6GV67Fi6d+b3/gmgqfntKpTsqjajaeusCssPZ7J97MjTolVLHJdzl4JErBvHMp5uZ2D+DosoR1Hl8fJCZyIyPN5IU3Zu+HeJ5c3khty8vJMEfxjDP5zyx82F48kwc7ftBeQEdzG62O6P2n/e000ez47Vk0v73Sxw7vkaSOsMpNx7z4dt7Fj1D2o7llM77O8mXP97czQd3NV7jwBGdAqXW4wQlKvHY7wsgDXql1HGLDnfxkzHdAeiSFru//I6JB56AdUaPNCbnZfLyl9uJjbiUnywzXF82n+yqLzHiIgVwNHj4+eg+HfhRwgPctPcfDP3yGcLxwKaFkN4XOo+AbmcdPtPHGGSNNQU0dv1rUHsvRCU1W7sBcNdQQwRh0XEA1FWXE6VBr5Rqq4Z3Td3/WMXPBmQw/eOJLNu6F5fxMtn/Lr27XsRQ+9gwp4Nnfn4Jry4dylWzV3NnykKuzf8Xkj8P+XQ6dBwCuVfA4Kut2Tn1FXjrq0it385zvnFMdX6A+73fE37hI+BovluExFtDHZGERVpBX1NVQVTqMd4UYBr0SqkWcXq3VE7vZiWiMYaq+vHERhwcQWFOB1ee2plwp4OH5kfxsOc0/Dj5fYelnFn6Lu3f+TXeD+7C5bWmNbqAehNG7fD/Y8Zn4dy08gX25H+OIyOXlHN/3yyPXRRPDXUSgS+hEwDewpWQ3b/JP6cpadArpVqciBAXefTHm0zOy2JyXha7K+p45MN8pq+N5e7a0xkly7iwfgHv+YbyjckhxVlHSmoaj4zLY2bUPTz5ZUd6Vy1m8IZ38D36Fo6cEUin06B9f+srIevA8I8x1hO6XOEHf3jJRihaA93HH74P6yEp9RJJZPYp7P4kEf+3s+H0Hzblj6fJiTneR441g7y8PLN06dJAV0Mp1cp5fX7yi6rYVlpDl9QYurWLRQ4Zt6+q93L3S/PptukFxrpW0oUCHFg55w6LR9r3J6xDf9j6OexejUnKhood+Nr1p97tJqZ4JQA1mSOJnjwDEjIPOv+6v43C566j152LeePPV3Ch+ZDw2zZDbak1A6jLmS3yswAQkWXGmLxjHac9eqVU0HA5HfTOiKd3RvxRj4mNcPHXqeOYs6IfMzaVsHrzDqL2rqePYyt9vFvps20bvbY/y15nChtSplBfvJFCbzdOLViHDwf/81+BzxHJbQXPwfS+kNoTup8N3cZCVREp7h0UOjrgdAjuXhcSvuZdauf8kqgtH0FNMUx8EPpeYi0VERZ11Hq2JO3RK6VCXlFlHbvKrRUy3/i6ELfHy6cbSthZUce5/TMYkJmIAdLiIhjZLdX6V8Gzc8guXcTZ4d8y2KwmHOvh6V6czI6/kkm3PsqG3RUse/QqJrk+ocKVSkVMZzqXf3XggzNyof9kqN4DznDoMgo6n24NH/m81vN9I+NPeKZQY3v0GvRKqTbJ5ze4vX6iwo98s1Odx8f0ed9RUFZLpKeC2s1fsMfZjrX1aVw5vBu321NJF6wrYvbHS8gv87OpzM94x1d0ianH5a5kkmMh6f7deMWFGD9O/PjD4zFRiTgqdyB+L5VnP0jc6TecUBs06JVSqhkYYw67LrDPtpIa3lxeyLpdFSRGh/FF/m6q9hZRF5FCr1QXHXd+yBDHd8RLDbud6WzwpDL0zPOZPO7ExvV1jF4ppZrB0UIerJVBfz62+/7XPr+hotZDYnQYIsJXWwazubiaHVVuNhdXcVrXFMb0TG/2OmvQK6VUM3E6hKSYA1M0h2YnMzQ7ucXroU+YUkqpEKdBr5RSIU6DXimlQpwGvVJKhTgNeqWUCnEa9EopFeI06JVSKsRp0CulVIhrFUsgiMgeYOsxDksFilugOq1NW203tN22t9V2Q9tt+4m2u7MxJu1YB7WKoG8MEVnamDUdQk1bbTe03ba31XZD2217c7dbh26UUirEadArpVSIC6agfyrQFQiQttpuaLttb6vthrbb9mZtd9CM0SullDoxwdSjV0opdQI06JVSKsQFRdCLyAQRWS8iG0TktkDX52SJyDMiUiQiqxuUJYvIPBHJt78n2eUiIo/YbV8lIoMbvGeqfXy+iEwNRFuOh4hkicgCEVkjIt+KyM/t8pBuu4hEisiXIrLSbvc9dnmOiCyx2zdLRMLt8gj79QZ7f3aDc91ul68XkfGBadHxERGniCwXkbn267bS7i0i8o2IrBCRpXZZYH7XjTGt+gtwAhuBLkA4sBLoE+h6nWSbzgAGA6sblN0P3GZv3wb8zd6eCLwLCDAMWGKXJwOb7O9J9nZSoNt2jHZnAIPt7TjgO6BPqLfdrn+svR0GLLHb8ypwuV0+A7jZ3r4FmGFvXw7Msrf72L//EUCO/f+FM9Dta0T7bwX+A8y1X7eVdm8BUg8pC8jvesB/GI34YZ0GvN/g9e3A7YGuVxO0K/uQoF8PZNjbGcB6e/tJ4IpDjwOuAJ5sUH7QccHwBcwBzm5LbQeiga+BU7HuhHTZ5ft/z4H3gdPsbZd9nBz6u9/wuNb6BWQCHwJjgLl2O0K+3XY9jxT0AfldD4ahm47A9gavC+yyUJNujNlpb+8C9j0x+GjtD+qfi/3P8kFYvduQb7s9fLECKALmYfVKy4wxXvuQhm3Y3z57fzmQQhC2G3gI+A3gt1+n0DbaDWCAD0RkmYhMs8sC8ruuDwdvhYwxRkRCdt6riMQCrwO/MMZUiMj+faHadmOMDxgoIonAm0CvAFep2YnIeUCRMWaZiIwKdH0CYIQxplBE2gHzRGRdw50t+bseDD36QiCrwetMuyzU7BaRDAD7e5FdfrT2B+XPRUTCsEL+JWPMG3Zxm2g7gDGmDFiANWSRKCL7OlsN27C/ffb+BKCE4Gv36cAFIrIFeAVr+OZhQr/dABhjCu3vRVh/3E8hQL/rwRD0XwHd7Sv14VgXad4KcJ2aw1vAvivqU7HGr/eVX21flR8GlNv/9HsfGCciSfaV+3F2WaslVtd9JrDWGPOPBrtCuu0ikmb35BGRKKzrEmuxAn+Sfdih7d7385gEfGSsAdq3gMvt2Sk5QHfgy5ZpxfEzxtxujMk0xmRj/X/7kTHmSkK83QAiEiMicfu2sX5HVxOo3/VAX7Bo5EWNiVgzNDYCdwa6Pk3QnpeBnYAHa8ztOqyxyA+BfGA+kGwfK8A/7bZ/A+Q1OM+PgA3217WBblcj2j0Ca9xyFbDC/poY6m0HBgDL7XavBu6yy7tgBdYG4L9AhF0eab/eYO/v0uBcd9o/j/XAOYFu23H8DEZxYNZNyLfbbuNK++vbfbkVqN91XQJBKaVCXDAM3SillDoJGvRKKRXiNOiVUirEadArpVSI06BXSqkQp0GvlE1EfiEi0YGuh1JNTadXKmWz7+DMM8YUB7ouSjUl7dGrNsm+c/Fte4341SLyB6ADsEBEFtjHjBORxSLytYj8116jZ9864/fba41/KSLdAtkWpY5Fg161VROAHcaYXGNMP6xVFncAo40xo0UkFfgdMNYYMxhYirWu+j7lxpj+wGP2e5VqtTToVVv1DXC2iPxNREYaY8oP2T8M64EXn9nLC08FOjfY/3KD76c1e22VOgm6TLFqk4wx39mPa5sI/FlEPjzkEAHmGWOuONopjrKtVKujPXrVJolIB6DGGPMi8ADWox0rsR5xCPAFcPq+8Xd7TL9Hg1NMafB9ccvUWqkToz161Vb1Bx4QET/WKqI3Yw3BvCciO+xx+muAl0Ukwn7P77BWUQVIEpFVQP3/t3cHJwDAMAzEuv/UeXSC/ppDGsKYYMi5797gW+aV8MgMk22cbgDiNHqAOI0eIE7QA8QJeoA4QQ8QJ+gB4gZyQGqRJrySPAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot loss\n",
    "plt.figure()\n",
    "plt.plot(losses)\n",
    "plt.title(\"Total loss\")\n",
    "plt.xlabel(\"step\")\n",
    "plt.savefig(loss_fig)\n",
    "plt.show()\n",
    "\n",
    "# plot perplexity\n",
    "plt.figure()\n",
    "if len(perps) > len(steps):\n",
    "    perps.pop()\n",
    "plt.plot(steps[5:], perps[5:], label=\"train\")\n",
    "if dev_source_data is not None:\n",
    "    plt.plot(steps[5:], dev_perps[5:], label=\"dev\")\n",
    "plt.title(\"Perplexity\")\n",
    "plt.xlabel(\"step\")\n",
    "plt.legend()\n",
    "plt.savefig(perp_fig)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading inference data ...\n",
      "\tDone.\n"
     ]
    }
   ],
   "source": [
    "print(\"Loading inference data ...\")\n",
    "\n",
    "# id_0, id_1, id_2 preserved for SOS, EOS, constant zero padding\n",
    "embed_shift = 3\n",
    "filename = config[\"inference\"][\"infer_source_file\"]\n",
    "max_leng = config[\"inference\"][\"infer_source_max_length\"]\n",
    "source_data = loadfile(filename, is_source=True,\n",
    "                       max_length=max_leng) + embed_shift\n",
    "print(\"\\tDone.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start inferring ...\n",
      "\tDone.\n"
     ]
    }
   ],
   "source": [
    "# Inference\n",
    "print(\"Start inferring ...\")\n",
    "final_result = []\n",
    "n_data = source_data.shape[0]\n",
    "n_pad = n_data % infer_batch_size\n",
    "if n_pad > 0:\n",
    "    n_pad = infer_batch_size - n_pad\n",
    "\n",
    "pad = np.zeros((n_pad, max_leng), dtype=np.int32)\n",
    "source_data = np.concatenate((source_data, pad))\n",
    "\n",
    "for ith in range(int(len(source_data) / infer_batch_size)):\n",
    "    start = ith * infer_batch_size\n",
    "    end = (ith + 1) * infer_batch_size\n",
    "    batch = source_data[start:end]\n",
    "\n",
    "    result = sess.run(infer_outputs, feed_dict={source_ids: batch})\n",
    "    result = result.ids[:, :, 0]\n",
    "\n",
    "    if result.shape[1] < max_iter:\n",
    "        l_pad = max_iter - result.shape[1]\n",
    "        result = np.concatenate(\n",
    "            (result, np.ones((infer_batch_size, l_pad))), axis=1)\n",
    "\n",
    "    final_result.append(result)\n",
    "\n",
    "final_result = np.concatenate(final_result)[:n_data] - embed_shift\n",
    "final_result[final_result < 0] = -1\n",
    "final_result = final_result.astype(str).tolist()\n",
    "final_result = list(map(lambda t: \" \".join(t), final_result))\n",
    "\n",
    "df = pd.DataFrame(data={\"0\": final_result})\n",
    "df.to_csv(config[\"inference\"][\"output_path\"], header=None, index=None)\n",
    "print(\"\\tDone.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14.329314931203088"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# perplexity\n",
    "random_indexes = np.random.choice(len(dev_source_data), 256)\n",
    "s = dev_source_data[random_indexes]\n",
    "t = dev_target_data[random_indexes]\n",
    "m = (t != -1)\n",
    "\n",
    "feed_dict = {\n",
    "    source_ids: s,\n",
    "    target_ids: t,\n",
    "    sequence_mask: m,\n",
    "}\n",
    "\n",
    "compute_perplexity(sess, CE, m, feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
